From 95635d5b9ce17b3c417e3869c9586181ede6f384 Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 12:01:54 +0100 Subject: [PATCH 001/374] code to produce models --- train/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 train/.gitkeep diff --git a/train/.gitkeep b/train/.gitkeep new file mode 100644 index 0000000..e69de29 From 4601237427f8b8cc2786a3bf845dbec7dfbd289d Mon Sep 17 00:00:00 2001 From: b-vr103 Date: Thu, 5 Dec 2019 12:10:55 +0100 Subject: [PATCH 002/374] add files needed for training --- train/__init__.py | 0 train/metrics.py | 338 ++++++++++++++++++++++++++++++++++++++++++++++ train/models.py | 317 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 655 insertions(+) create mode 100644 train/__init__.py create mode 100644 train/metrics.py create mode 100644 train/models.py diff --git a/train/__init__.py b/train/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/metrics.py b/train/metrics.py new file mode 100644 index 0000000..c63cc22 --- /dev/null +++ b/train/metrics.py @@ -0,0 +1,338 @@ +from keras import backend as K +import tensorflow as tf +import numpy as np + +def focal_loss(gamma=2., alpha=4.): + + gamma = float(gamma) + alpha = float(alpha) + + def focal_loss_fixed(y_true, y_pred): + """Focal loss for multi-classification + FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t) + Notice: y_pred is probability after softmax + gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper + d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x) + Focal Loss for Dense Object Detection + https://arxiv.org/abs/1708.02002 + + Arguments: + y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls] + y_pred {tensor} -- model's output, shape of [batch_size, num_cls] + + Keyword Arguments: + gamma {float} -- (default: {2.0}) + alpha {float} -- (default: {4.0}) + + Returns: + [tensor] -- loss. + """ + epsilon = 1.e-9 + y_true = tf.convert_to_tensor(y_true, tf.float32) + y_pred = tf.convert_to_tensor(y_pred, tf.float32) + + model_out = tf.add(y_pred, epsilon) + ce = tf.multiply(y_true, -tf.log(model_out)) + weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma)) + fl = tf.multiply(alpha, tf.multiply(weight, ce)) + reduced_fl = tf.reduce_max(fl, axis=1) + return tf.reduce_mean(reduced_fl) + return focal_loss_fixed + +def weighted_categorical_crossentropy(weights=None): + """ weighted_categorical_crossentropy + + Args: + * weights: crossentropy weights + Returns: + * weighted categorical crossentropy function + """ + + def loss(y_true, y_pred): + labels_floats = tf.cast(y_true, tf.float32) + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) + + if weights is not None: + weight_mask = tf.maximum(tf.reduce_max(tf.constant( + np.array(weights, dtype=np.float32)[None, None, None]) + * labels_floats, axis=-1), 1.0) + per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] + return tf.reduce_mean(per_pixel_loss) + return loss +def image_categorical_cross_entropy(y_true, y_pred, weights=None): + """ + :param y_true: tensor of shape (batch_size, height, width) representing the ground truth. + :param y_pred: tensor of shape (batch_size, height, width) representing the prediction. + :return: The mean cross-entropy on softmaxed tensors. + """ + + labels_floats = tf.cast(y_true, tf.float32) + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) + + if weights is not None: + weight_mask = tf.maximum( + tf.reduce_max(tf.constant( + np.array(weights, dtype=np.float32)[None, None, None]) + * labels_floats, axis=-1), 1.0) + per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] + + return tf.reduce_mean(per_pixel_loss) +def class_tversky(y_true, y_pred): + smooth = 1.0#1.00 + + y_true = K.permute_dimensions(y_true, (3,1,2,0)) + y_pred = K.permute_dimensions(y_pred, (3,1,2,0)) + + y_true_pos = K.batch_flatten(y_true) + y_pred_pos = K.batch_flatten(y_pred) + true_pos = K.sum(y_true_pos * y_pred_pos, 1) + false_neg = K.sum(y_true_pos * (1-y_pred_pos), 1) + false_pos = K.sum((1-y_true_pos)*y_pred_pos, 1) + alpha = 0.2#0.5 + beta=0.8 + return (true_pos + smooth)/(true_pos + alpha*false_neg + (beta)*false_pos + smooth) + +def focal_tversky_loss(y_true,y_pred): + pt_1 = class_tversky(y_true, y_pred) + gamma =1.3#4./3.0#1.3#4.0/3.00# 0.75 + return K.sum(K.pow((1-pt_1), gamma)) + +def generalized_dice_coeff2(y_true, y_pred): + n_el = 1 + for dim in y_true.shape: + n_el *= int(dim) + n_cl = y_true.shape[-1] + w = K.zeros(shape=(n_cl,)) + w = (K.sum(y_true, axis=(0,1,2)))/(n_el) + w = 1/(w**2+0.000001) + numerator = y_true*y_pred + numerator = w*K.sum(numerator,(0,1,2)) + numerator = K.sum(numerator) + denominator = y_true+y_pred + denominator = w*K.sum(denominator,(0,1,2)) + denominator = K.sum(denominator) + return 2*numerator/denominator +def generalized_dice_coeff(y_true, y_pred): + axes = tuple(range(1, len(y_pred.shape)-1)) + Ncl = y_pred.shape[-1] + w = K.zeros(shape=(Ncl,)) + w = K.sum(y_true, axis=axes) + w = 1/(w**2+0.000001) + # Compute gen dice coef: + numerator = y_true*y_pred + numerator = w*K.sum(numerator,axes) + numerator = K.sum(numerator) + + denominator = y_true+y_pred + denominator = w*K.sum(denominator,axes) + denominator = K.sum(denominator) + + gen_dice_coef = 2*numerator/denominator + + return gen_dice_coef + +def generalized_dice_loss(y_true, y_pred): + return 1 - generalized_dice_coeff2(y_true, y_pred) +def soft_dice_loss(y_true, y_pred, epsilon=1e-6): + ''' + Soft dice loss calculation for arbitrary batch size, number of classes, and number of spatial dimensions. + Assumes the `channels_last` format. + + # Arguments + y_true: b x X x Y( x Z...) x c One hot encoding of ground truth + y_pred: b x X x Y( x Z...) x c Network output, must sum to 1 over c channel (such as after softmax) + epsilon: Used for numerical stability to avoid divide by zero errors + + # References + V-Net: Fully Convolutional Neural Networks for Volumetric Medical Image Segmentation + https://arxiv.org/abs/1606.04797 + More details on Dice loss formulation + https://mediatum.ub.tum.de/doc/1395260/1395260.pdf (page 72) + + Adapted from https://github.com/Lasagne/Recipes/issues/99#issuecomment-347775022 + ''' + + # skip the batch and class axis for calculating Dice score + axes = tuple(range(1, len(y_pred.shape)-1)) + + numerator = 2. * K.sum(y_pred * y_true, axes) + + denominator = K.sum(K.square(y_pred) + K.square(y_true), axes) + return 1.00 - K.mean(numerator / (denominator + epsilon)) # average over classes and batch + +def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last = True, mean_per_class=False, verbose=False): + """ + Compute mean metrics of two segmentation masks, via Keras. + + IoU(A,B) = |A & B| / (| A U B|) + Dice(A,B) = 2*|A & B| / (|A| + |B|) + + Args: + y_true: true masks, one-hot encoded. + y_pred: predicted masks, either softmax outputs, or one-hot encoded. + metric_name: metric to be computed, either 'iou' or 'dice'. + metric_type: one of 'standard' (default), 'soft', 'naive'. + In the standard version, y_pred is one-hot encoded and the mean + is taken only over classes that are present (in y_true or y_pred). + The 'soft' version of the metrics are computed without one-hot + encoding y_pred. + The 'naive' version return mean metrics where absent classes contribute + to the class mean as 1.0 (instead of being dropped from the mean). + drop_last = True: boolean flag to drop last class (usually reserved + for background class in semantic segmentation) + mean_per_class = False: return mean along batch axis for each class. + verbose = False: print intermediate results such as intersection, union + (as number of pixels). + Returns: + IoU/Dice of y_true and y_pred, as a float, unless mean_per_class == True + in which case it returns the per-class metric, averaged over the batch. + + Inputs are B*W*H*N tensors, with + B = batch size, + W = width, + H = height, + N = number of classes + """ + + flag_soft = (metric_type == 'soft') + flag_naive_mean = (metric_type == 'naive') + + # always assume one or more classes + num_classes = K.shape(y_true)[-1] + + if not flag_soft: + # get one-hot encoded masks from y_pred (true masks should already be one-hot) + y_pred = K.one_hot(K.argmax(y_pred), num_classes) + y_true = K.one_hot(K.argmax(y_true), num_classes) + + # if already one-hot, could have skipped above command + # keras uses float32 instead of float64, would give error down (but numpy arrays or keras.to_categorical gives float64) + y_true = K.cast(y_true, 'float32') + y_pred = K.cast(y_pred, 'float32') + + # intersection and union shapes are batch_size * n_classes (values = area in pixels) + axes = (1,2) # W,H axes of each image + intersection = K.sum(K.abs(y_true * y_pred), axis=axes) + mask_sum = K.sum(K.abs(y_true), axis=axes) + K.sum(K.abs(y_pred), axis=axes) + union = mask_sum - intersection # or, np.logical_or(y_pred, y_true) for one-hot + + smooth = .001 + iou = (intersection + smooth) / (union + smooth) + dice = 2 * (intersection + smooth)/(mask_sum + smooth) + + metric = {'iou': iou, 'dice': dice}[metric_name] + + # define mask to be 0 when no pixels are present in either y_true or y_pred, 1 otherwise + mask = K.cast(K.not_equal(union, 0), 'float32') + + if drop_last: + metric = metric[:,:-1] + mask = mask[:,:-1] + + if verbose: + print('intersection, union') + print(K.eval(intersection), K.eval(union)) + print(K.eval(intersection/union)) + + # return mean metrics: remaining axes are (batch, classes) + if flag_naive_mean: + return K.mean(metric) + + # take mean only over non-absent classes + class_count = K.sum(mask, axis=0) + non_zero = tf.greater(class_count, 0) + non_zero_sum = tf.boolean_mask(K.sum(metric * mask, axis=0), non_zero) + non_zero_count = tf.boolean_mask(class_count, non_zero) + + if verbose: + print('Counts of inputs with class present, metrics for non-absent classes') + print(K.eval(class_count), K.eval(non_zero_sum / non_zero_count)) + + return K.mean(non_zero_sum / non_zero_count) + +def mean_iou(y_true, y_pred, **kwargs): + """ + Compute mean Intersection over Union of two segmentation masks, via Keras. + + Calls metrics_k(y_true, y_pred, metric_name='iou'), see there for allowed kwargs. + """ + return seg_metrics(y_true, y_pred, metric_name='iou', **kwargs) +def Mean_IOU(y_true, y_pred): + nb_classes = K.int_shape(y_pred)[-1] + iou = [] + true_pixels = K.argmax(y_true, axis=-1) + pred_pixels = K.argmax(y_pred, axis=-1) + void_labels = K.equal(K.sum(y_true, axis=-1), 0) + for i in range(0, nb_classes): # exclude first label (background) and last label (void) + true_labels = K.equal(true_pixels, i)# & ~void_labels + pred_labels = K.equal(pred_pixels, i)# & ~void_labels + inter = tf.to_int32(true_labels & pred_labels) + union = tf.to_int32(true_labels | pred_labels) + legal_batches = K.sum(tf.to_int32(true_labels), axis=1)>0 + ious = K.sum(inter, axis=1)/K.sum(union, axis=1) + iou.append(K.mean(tf.gather(ious, indices=tf.where(legal_batches)))) # returns average IoU of the same objects + iou = tf.stack(iou) + legal_labels = ~tf.debugging.is_nan(iou) + iou = tf.gather(iou, indices=tf.where(legal_labels)) + return K.mean(iou) + +def iou_vahid(y_true, y_pred): + nb_classes = tf.shape(y_true)[-1]+tf.to_int32(1) + true_pixels = K.argmax(y_true, axis=-1) + pred_pixels = K.argmax(y_pred, axis=-1) + iou = [] + + for i in tf.range(nb_classes): + tp=K.sum( tf.to_int32( K.equal(true_pixels, i) & K.equal(pred_pixels, i) ) ) + fp=K.sum( tf.to_int32( K.not_equal(true_pixels, i) & K.equal(pred_pixels, i) ) ) + fn=K.sum( tf.to_int32( K.equal(true_pixels, i) & K.not_equal(pred_pixels, i) ) ) + iouh=tp/(tp+fp+fn) + iou.append(iouh) + return K.mean(iou) + + +def IoU_metric(Yi,y_predi): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + y_predi = np.argmax(y_predi, axis=3) + y_testi = np.argmax(Yi, axis=3) + IoUs = [] + Nclass = int(np.max(Yi)) + 1 + for c in range(Nclass): + TP = np.sum( (Yi == c)&(y_predi==c) ) + FP = np.sum( (Yi != c)&(y_predi==c) ) + FN = np.sum( (Yi == c)&(y_predi != c)) + IoU = TP/float(TP + FP + FN) + IoUs.append(IoU) + return K.cast( np.mean(IoUs) ,dtype='float32' ) + + +def IoU_metric_keras(y_true, y_pred): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + init = tf.global_variables_initializer() + sess = tf.Session() + sess.run(init) + + return IoU_metric(y_true.eval(session=sess), y_pred.eval(session=sess)) + +def jaccard_distance_loss(y_true, y_pred, smooth=100): + """ + Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|) + = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|)) + + The jaccard distance loss is usefull for unbalanced datasets. This has been + shifted so it converges on 0 and is smoothed to avoid exploding or disapearing + gradient. + + Ref: https://en.wikipedia.org/wiki/Jaccard_index + + @url: https://gist.github.com/wassname/f1452b748efcbeb4cb9b1d059dce6f96 + @author: wassname + """ + intersection = K.sum(K.abs(y_true * y_pred), axis=-1) + sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1) + jac = (intersection + smooth) / (sum_ - intersection + smooth) + return (1 - jac) * smooth + + diff --git a/train/models.py b/train/models.py new file mode 100644 index 0000000..7c806b4 --- /dev/null +++ b/train/models.py @@ -0,0 +1,317 @@ +from keras.models import * +from keras.layers import * +from keras import layers +from keras.regularizers import l2 + +resnet50_Weights_path='./pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' +IMAGE_ORDERING ='channels_last' +MERGE_AXIS=-1 + + +def one_side_pad( x ): + x = ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING)(x) + if IMAGE_ORDERING == 'channels_first': + x = Lambda(lambda x : x[: , : , :-1 , :-1 ] )(x) + elif IMAGE_ORDERING == 'channels_last': + x = Lambda(lambda x : x[: , :-1 , :-1 , : ] )(x) + return x + +def identity_block(input_tensor, kernel_size, filters, stage, block): + """The identity block is the block that has no conv layer at shortcut. + # Arguments + input_tensor: input tensor + kernel_size: defualt 3, the kernel size of middle conv layer at main path + filters: list of integers, the filterss of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + # Returns + Output tensor for the block. + """ + filters1, filters2, filters3 = filters + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = Conv2D(filters1, (1, 1) , data_format=IMAGE_ORDERING , name=conv_name_base + '2a')(input_tensor) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) + x = Activation('relu')(x) + + x = Conv2D(filters2, kernel_size , data_format=IMAGE_ORDERING , + padding='same', name=conv_name_base + '2b')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) + x = Activation('relu')(x) + + x = Conv2D(filters3 , (1, 1), data_format=IMAGE_ORDERING , name=conv_name_base + '2c')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) + + x = layers.add([x, input_tensor]) + x = Activation('relu')(x) + return x + + +def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): + """conv_block is the block that has a conv layer at shortcut + # Arguments + input_tensor: input tensor + kernel_size: defualt 3, the kernel size of middle conv layer at main path + filters: list of integers, the filterss of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + # Returns + Output tensor for the block. + Note that from stage 3, the first conv layer at main path is with strides=(2,2) + And the shortcut should have strides=(2,2) as well + """ + filters1, filters2, filters3 = filters + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = Conv2D(filters1, (1, 1) , data_format=IMAGE_ORDERING , strides=strides, + name=conv_name_base + '2a')(input_tensor) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) + x = Activation('relu')(x) + + x = Conv2D(filters2, kernel_size , data_format=IMAGE_ORDERING , padding='same', + name=conv_name_base + '2b')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) + x = Activation('relu')(x) + + x = Conv2D(filters3, (1, 1) , data_format=IMAGE_ORDERING , name=conv_name_base + '2c')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) + + shortcut = Conv2D(filters3, (1, 1) , data_format=IMAGE_ORDERING , strides=strides, + name=conv_name_base + '1')(input_tensor) + shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut) + + x = layers.add([x, shortcut]) + x = Activation('relu')(x) + return x + + +def resnet50_unet_light(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + assert input_height%32 == 0 + assert input_width%32 == 0 + + + img_input = Input(shape=(input_height,input_width , 3 )) + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) + + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x ) + + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + + if pretraining: + model=Model( img_input , x ).load_weights(resnet50_Weights_path) + + + v512_2048 = Conv2D( 512 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f5 ) + v512_2048 = ( BatchNormalization(axis=bn_axis))(v512_2048) + v512_2048 = Activation('relu')(v512_2048) + + + + v512_1024=Conv2D( 512 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f4 ) + v512_1024 = ( BatchNormalization(axis=bn_axis))(v512_1024) + v512_1024 = Activation('relu')(v512_1024) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(v512_2048) + o = ( concatenate([ o ,v512_1024],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([ o ,f3],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f2],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 128 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay) ) )(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f1],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 64 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,img_input],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 32 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + + o = Conv2D( n_classes , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( o ) + o = ( BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + + + model = Model( img_input , o ) + return model + +def resnet50_unet(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + assert input_height%32 == 0 + assert input_width%32 == 0 + + + img_input = Input(shape=(input_height,input_width , 3 )) + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) + + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x ) + + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + if pretraining: + Model( img_input , x ).load_weights(resnet50_Weights_path) + + v1024_2048 = Conv2D( 1024 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f5 ) + v1024_2048 = ( BatchNormalization(axis=bn_axis))(v1024_2048) + v1024_2048 = Activation('relu')(v1024_2048) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(v1024_2048) + o = ( concatenate([ o ,f4],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([ o ,f3],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f2],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 128 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay) ) )(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f1],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 64 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,img_input],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 32 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = Conv2D( n_classes , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( o ) + o = ( BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + + model = Model( img_input , o ) + + + + + return model From 226330535d0d01c67e4c18c7957e3d69b8f5f672 Mon Sep 17 00:00:00 2001 From: b-vr103 Date: Thu, 5 Dec 2019 14:05:07 +0100 Subject: [PATCH 003/374] add files needed for training --- train/README | 23 +++ train/config_params.json | 24 +++ train/train.py | 192 ++++++++++++++++++++++ train/utils.py | 336 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 575 insertions(+) create mode 100644 train/README create mode 100644 train/config_params.json create mode 100644 train/train.py create mode 100644 train/utils.py diff --git a/train/README b/train/README new file mode 100644 index 0000000..7d8d790 --- /dev/null +++ b/train/README @@ -0,0 +1,23 @@ +how to train: + just run: python train.py with config_params.json + + +format of ground truth: + + Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and labels should be 0 and 1 for each class and pixel. + In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels by pixels from 0 , 1 ,2 .., n_classes-1. + The labels format should be png. + + If you have an image label for binary case it should look like this: + + Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. + +traing , evaluation and output: + train and evaluation folder should have subfolder of images and labels. + And output folder should be free folder which the output model will be written there. + +patches: + + if you want to train your model with patches, the height and width of patches should be defined and also number of batchs (how many patches should be seen by model by each iteration). + In the case that model should see the image once, like page extraction, the patches should be set to false. + diff --git a/train/config_params.json b/train/config_params.json new file mode 100644 index 0000000..52db6db --- /dev/null +++ b/train/config_params.json @@ -0,0 +1,24 @@ +{ + "n_classes" : 2, + "n_epochs" : 2, + "input_height" : 448, + "input_width" : 896, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : false, + "flip_aug" : false, + "elastic_aug" : false, + "blur_aug" : false, + "scaling" : false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "rotation": false, + "weighted_loss": true, + "dir_train": "/home/vahid/textline_gt_images/train_light", + "dir_eval": "/home/vahid/textline_gt_images/eval", + "dir_output": "/home/vahid/textline_gt_images/output" +} diff --git a/train/train.py b/train/train.py new file mode 100644 index 0000000..07c7418 --- /dev/null +++ b/train/train.py @@ -0,0 +1,192 @@ +import os +import sys +import tensorflow as tf +from keras.backend.tensorflow_backend import set_session +import keras , warnings +from keras.optimizers import * +from sacred import Experiment +from models import * +from utils import * +from metrics import * + + +def configuration(): + keras.backend.clear_session() + tf.reset_default_graph() + warnings.filterwarnings('ignore') + + os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' + config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) + + + config.gpu_options.allow_growth = True + config.gpu_options.per_process_gpu_memory_fraction=0.95#0.95 + config.gpu_options.visible_device_list="0" + set_session(tf.Session(config=config)) + +def get_dirs_or_files(input_data): + if os.path.isdir(input_data): + image_input, labels_input = os.path.join(input_data, 'images/'), os.path.join(input_data, 'labels/') + # Check if training dir exists + assert os.path.isdir(image_input), "{} is not a directory".format(image_input) + assert os.path.isdir(labels_input), "{} is not a directory".format(labels_input) + return image_input, labels_input + +ex = Experiment() + +@ex.config +def config_params(): + n_classes=None # Number of classes. If your case study is binary case the set it to 2 and otherwise give your number of cases. + n_epochs=1 + input_height=224*1 + input_width=224*1 + weight_decay=1e-6 # Weight decay of l2 regularization of model layers. + n_batch=1 # Number of batches at each iteration. + learning_rate=1e-4 + patches=False # Make patches of image in order to use all information of image. In the case of page + # extraction this should be set to false since model should see all image. + augmentation=False + flip_aug=False # Flip image (augmentation). + elastic_aug=False # Elastic transformation (augmentation). + blur_aug=False # Blur patches of image (augmentation). + scaling=False # Scaling of patches (augmentation) will be imposed if this set to true. + binarization=False # Otsu thresholding. Used for augmentation in the case of binary case like textline prediction. For multicases should not be applied. + dir_train=None # Directory of training dataset (sub-folders should be named images and labels). + dir_eval=None # Directory of validation dataset (sub-folders should be named images and labels). + dir_output=None # Directory of output where the model should be saved. + pretraining=False # Set true to load pretrained weights of resnet50 encoder. + weighted_loss=False # Set True if classes are unbalanced and you want to use weighted loss function. + scaling_bluring=False + rotation: False + scaling_binarization=False + blur_k=['blur','guass','median'] # Used in order to blur image. Used for augmentation. + scales=[0.9 , 1.1 ] # Scale patches with these scales. Used for augmentation. + flip_index=[0,1] # Flip image. Used for augmentation. + + +@ex.automain +def run(n_classes,n_epochs,input_height, + input_width,weight_decay,weighted_loss, + n_batch,patches,augmentation,flip_aug,blur_aug,scaling, binarization, + blur_k,scales,dir_train, + scaling_bluring,scaling_binarization,rotation, + flip_index,dir_eval ,dir_output,pretraining,learning_rate): + + dir_img,dir_seg=get_dirs_or_files(dir_train) + dir_img_val,dir_seg_val=get_dirs_or_files(dir_eval) + + # make first a directory in output for both training and evaluations in order to flow data from these directories. + dir_train_flowing=os.path.join(dir_output,'train') + dir_eval_flowing=os.path.join(dir_output,'eval') + + dir_flow_train_imgs=os.path.join(dir_train_flowing,'images') + dir_flow_train_labels=os.path.join(dir_train_flowing,'labels') + + dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images') + dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels') + + if os.path.isdir(dir_train_flowing): + os.system('rm -rf '+dir_train_flowing) + os.makedirs(dir_train_flowing) + else: + os.makedirs(dir_train_flowing) + + if os.path.isdir(dir_eval_flowing): + os.system('rm -rf '+dir_eval_flowing) + os.makedirs(dir_eval_flowing) + else: + os.makedirs(dir_eval_flowing) + + + os.mkdir(dir_flow_train_imgs) + os.mkdir(dir_flow_train_labels) + + os.mkdir(dir_flow_eval_imgs) + os.mkdir(dir_flow_eval_labels) + + + + #set the gpu configuration + configuration() + + + #writing patches into a sub-folder in order to be flowed from directory. + provide_patches(dir_img,dir_seg,dir_flow_train_imgs, + dir_flow_train_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + augmentation=augmentation,patches=patches) + + provide_patches(dir_img_val,dir_seg_val,dir_flow_eval_imgs, + dir_flow_eval_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + augmentation=False,patches=patches) + + if weighted_loss: + weights=np.zeros(n_classes) + for obj in os.listdir(dir_seg): + label_obj=cv2.imread(dir_seg+'/'+obj) + label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) + weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + + + weights=1.00/weights + + weights=weights/float(np.sum(weights)) + weights=weights/float(np.min(weights)) + weights=weights/float(np.sum(weights)) + + + + + #get our model. + model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + + #if you want to see the model structure just uncomment model summary. + #model.summary() + + + if not weighted_loss: + model.compile(loss='categorical_crossentropy', + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + if weighted_loss: + model.compile(loss=weighted_categorical_crossentropy(weights), + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + + mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', + save_weights_only=True, period=1) + + + #generating train and evaluation data + train_gen = data_gen(dir_flow_train_imgs,dir_flow_train_labels, batch_size = n_batch, + input_height=input_height, input_width=input_width,n_classes=n_classes ) + val_gen = data_gen(dir_flow_eval_imgs,dir_flow_eval_labels, batch_size = n_batch, + input_height=input_height, input_width=input_width,n_classes=n_classes ) + + + model.fit_generator( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch), + validation_data=val_gen, + validation_steps=1, + epochs=n_epochs) + + + + os.system('rm -rf '+dir_train_flowing) + os.system('rm -rf '+dir_eval_flowing) + + model.save(dir_output+'/'+'model'+'.h5') + + + + + + + + + + diff --git a/train/utils.py b/train/utils.py new file mode 100644 index 0000000..afdc9e5 --- /dev/null +++ b/train/utils.py @@ -0,0 +1,336 @@ +import os +import cv2 +import numpy as np +import seaborn as sns +from scipy.ndimage.interpolation import map_coordinates +from scipy.ndimage.filters import gaussian_filter +import random +from tqdm import tqdm + + + + +def bluring(img_in,kind): + if kind=='guass': + img_blur = cv2.GaussianBlur(img_in,(5,5),0) + elif kind=="median": + img_blur = cv2.medianBlur(img_in,5) + elif kind=='blur': + img_blur=cv2.blur(img_in,(5,5)) + return img_blur + +def color_images(seg, n_classes): + ann_u=range(n_classes) + if len(np.shape(seg))==3: + seg=seg[:,:,0] + + seg_img=np.zeros((np.shape(seg)[0],np.shape(seg)[1],3)).astype(float) + colors=sns.color_palette("hls", n_classes) + + for c in ann_u: + c=int(c) + segl=(seg==c) + seg_img[:,:,0]+=segl*(colors[c][0]) + seg_img[:,:,1]+=segl*(colors[c][1]) + seg_img[:,:,2]+=segl*(colors[c][2]) + return seg_img + + +def resize_image(seg_in,input_height,input_width): + return cv2.resize(seg_in,(input_width,input_height),interpolation=cv2.INTER_NEAREST) +def get_one_hot(seg,input_height,input_width,n_classes): + seg=seg[:,:,0] + seg_f=np.zeros((input_height, input_width,n_classes)) + for j in range(n_classes): + seg_f[:,:,j]=(seg==j).astype(int) + return seg_f + + +def IoU(Yi,y_predi): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + + IoUs = [] + classes_true=np.unique(Yi) + for c in classes_true: + TP = np.sum( (Yi == c)&(y_predi==c) ) + FP = np.sum( (Yi != c)&(y_predi==c) ) + FN = np.sum( (Yi == c)&(y_predi != c)) + IoU = TP/float(TP + FP + FN) + print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c,TP,FP,FN,IoU)) + IoUs.append(IoU) + mIoU = np.mean(IoUs) + print("_________________") + print("Mean IoU: {:4.3f}".format(mIoU)) + return mIoU +def data_gen(img_folder, mask_folder, batch_size,input_height, input_width,n_classes): + c = 0 + n = os.listdir(img_folder) #List of training images + random.shuffle(n) + while True: + img = np.zeros((batch_size, input_height, input_width, 3)).astype('float') + mask = np.zeros((batch_size, input_height, input_width, n_classes)).astype('float') + + for i in range(c, c+batch_size): #initially from 0 to 16, c = 0. + #print(img_folder+'/'+n[i]) + filename=n[i].split('.')[0] + train_img = cv2.imread(img_folder+'/'+n[i])/255. + train_img = cv2.resize(train_img, (input_width, input_height),interpolation=cv2.INTER_NEAREST)# Read an image from folder and resize + + img[i-c] = train_img #add to array - img[0], img[1], and so on. + train_mask = cv2.imread(mask_folder+'/'+filename+'.png') + #print(mask_folder+'/'+filename+'.png') + #print(train_mask.shape) + train_mask = get_one_hot( resize_image(train_mask,input_height,input_width),input_height,input_width,n_classes) + #train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] + + mask[i-c] = train_mask + + c+=batch_size + if(c+batch_size>=len(os.listdir(img_folder))): + c=0 + random.shuffle(n) + yield img, mask + +def otsu_copy(img): + img_r=np.zeros(img.shape) + img1=img[:,:,0] + img2=img[:,:,1] + img3=img[:,:,2] + _, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold2 = cv2.threshold(img2, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold3 = cv2.threshold(img3, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + img_r[:,:,0]=threshold1 + img_r[:,:,1]=threshold1 + img_r[:,:,2]=threshold1 + return img_r + +def rotation_90(img): + img_rot=np.zeros((img.shape[1],img.shape[0],img.shape[2])) + img_rot[:,:,0]=img[:,:,0].T + img_rot[:,:,1]=img[:,:,1].T + img_rot[:,:,2]=img[:,:,2].T + return img_rot + +def get_patches(dir_img_f,dir_seg_f,img,label,height,width,indexer): + + + img_h=img.shape[0] + img_w=img.shape[1] + + nxf=img_w/float(width) + nyf=img_h/float(height) + + if nxf>int(nxf): + nxf=int(nxf)+1 + if nyf>int(nyf): + nyf=int(nyf)+1 + + nxf=int(nxf) + nyf=int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d=i*width + index_x_u=(i+1)*width + + index_y_d=j*height + index_y_u=(j+1)*height + + if index_x_u>img_w: + index_x_u=img_w + index_x_d=img_w-width + if index_y_u>img_h: + index_y_u=img_h + index_y_d=img_h-height + + + img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] + label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] + + cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) + cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) + indexer+=1 + return indexer + + + +def get_patches_num_scale(dir_img_f,dir_seg_f,img,label,height,width,indexer,scaler): + + + img_h=img.shape[0] + img_w=img.shape[1] + + height_scale=int(height*scaler) + width_scale=int(width*scaler) + + + nxf=img_w/float(width_scale) + nyf=img_h/float(height_scale) + + if nxf>int(nxf): + nxf=int(nxf)+1 + if nyf>int(nyf): + nyf=int(nyf)+1 + + nxf=int(nxf) + nyf=int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d=i*width_scale + index_x_u=(i+1)*width_scale + + index_y_d=j*height_scale + index_y_u=(j+1)*height_scale + + if index_x_u>img_w: + index_x_u=img_w + index_x_d=img_w-width_scale + if index_y_u>img_h: + index_y_u=img_h + index_y_d=img_h-height_scale + + + img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] + label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] + + img_patch=resize_image(img_patch,height,width) + label_patch=resize_image(label_patch,height,width) + + cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) + cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) + indexer+=1 + + return indexer + + + +def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, + dir_flow_train_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + augmentation=False,patches=False): + + imgs_cv_train=np.array(os.listdir(dir_img)) + segs_cv_train=np.array(os.listdir(dir_seg)) + + indexer=0 + for im, seg_i in tqdm(zip(imgs_cv_train,segs_cv_train)): + img_name=im.split('.')[0] + + if not patches: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', resize_image(cv2.imread(dir_img+'/'+im),input_height,input_width ) ) + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width ) ) + indexer+=1 + + if augmentation: + if rotation: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + rotation_90( resize_image(cv2.imread(dir_img+'/'+im), + input_height,input_width) ) ) + + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png', + rotation_90 ( resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width) ) ) + indexer+=1 + + if flip_aug: + for f_i in flip_index: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + resize_image(cv2.flip(cv2.imread(dir_img+'/'+im),f_i),input_height,input_width) ) + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , + resize_image(cv2.flip(cv2.imread(dir_seg+'/'+img_name+'.png'),f_i),input_height,input_width) ) + indexer+=1 + + if blur_aug: + for blur_i in blur_k: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + (resize_image(bluring(cv2.imread(dir_img+'/'+im),blur_i),input_height,input_width) ) ) + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , + resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width) ) + indexer+=1 + + + if binarization: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + resize_image(otsu_copy( cv2.imread(dir_img+'/'+im)),input_height,input_width )) + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png', + resize_image( cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width )) + indexer+=1 + + + + + + + if patches: + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) + + if augmentation: + + if rotation: + + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + rotation_90( cv2.imread(dir_img+'/'+im) ), + rotation_90( cv2.imread(dir_seg+'/'+img_name+'.png') ), + input_height,input_width,indexer=indexer) + if flip_aug: + for f_i in flip_index: + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + cv2.flip( cv2.imread(dir_img+'/'+im) , f_i), + cv2.flip( cv2.imread(dir_seg+'/'+img_name+'.png') ,f_i), + input_height,input_width,indexer=indexer) + if blur_aug: + for blur_i in blur_k: + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + bluring( cv2.imread(dir_img+'/'+im) , blur_i), + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) + + + if scaling: + for sc_ind in scales: + indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + cv2.imread(dir_img+'/'+im) , + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer,scaler=sc_ind) + if binarization: + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + otsu_copy( cv2.imread(dir_img+'/'+im)), + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) + + + + if scaling_bluring: + for sc_ind in scales: + for blur_i in blur_k: + indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + bluring( cv2.imread(dir_img+'/'+im) , blur_i) , + cv2.imread(dir_seg+'/'+img_name+'.png') , + input_height,input_width,indexer=indexer,scaler=sc_ind) + + if scaling_binarization: + for sc_ind in scales: + indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + otsu_copy( cv2.imread(dir_img+'/'+im)) , + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer,scaler=sc_ind) + + + + + + From 1882dd8f53b665993c806ff5587562772f65c8a7 Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 14:05:55 +0100 Subject: [PATCH 004/374] Update config_params.json --- train/config_params.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index 52db6db..5066444 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -18,7 +18,7 @@ "scaling_binarization" : false, "rotation": false, "weighted_loss": true, - "dir_train": "/home/vahid/textline_gt_images/train_light", - "dir_eval": "/home/vahid/textline_gt_images/eval", - "dir_output": "/home/vahid/textline_gt_images/output" + "dir_train": "../train", + "dir_eval": "../eval", + "dir_output": "../output" } From e8afb370bafa617250ef3f15fe35a721e0a1ccbd Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 14:08:08 +0100 Subject: [PATCH 005/374] Update README --- train/README | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/train/README b/train/README index 7d8d790..8d478bd 100644 --- a/train/README +++ b/train/README @@ -4,17 +4,20 @@ how to train: format of ground truth: - Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and labels should be 0 and 1 for each class and pixel. - In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels by pixels from 0 , 1 ,2 .., n_classes-1. + Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and + labels should be 0 and 1 for each class and pixel. + In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels + by pixels set from 0 , 1 ,2 .., n_classes-1. The labels format should be png. If you have an image label for binary case it should look like this: - Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. + Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] + this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. -traing , evaluation and output: +training , evaluation and output: train and evaluation folder should have subfolder of images and labels. - And output folder should be free folder which the output model will be written there. + And output folder should be empty folder which the output model will be written there. patches: From 99a02a1bf55a8022110ca78d0363c2eae610cecf Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 14:11:37 +0100 Subject: [PATCH 006/374] Update README --- train/README | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/train/README b/train/README index 8d478bd..54ea408 100644 --- a/train/README +++ b/train/README @@ -21,6 +21,7 @@ training , evaluation and output: patches: - if you want to train your model with patches, the height and width of patches should be defined and also number of batchs (how many patches should be seen by model by each iteration). + if you want to train your model with patches, the height and width of patches should be defined and also number of + batchs (how many patches should be seen by model by each iteration). In the case that model should see the image once, like page extraction, the patches should be set to false. From 7eb3dd26addb0131cf39c6bdbf0dcd88ed61d8d5 Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 16:11:31 +0100 Subject: [PATCH 007/374] Update README --- train/README | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/train/README b/train/README index 54ea408..e103b0b 100644 --- a/train/README +++ b/train/README @@ -1,8 +1,8 @@ -how to train: +# Train just run: python train.py with config_params.json -format of ground truth: +# Ground truth format Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and labels should be 0 and 1 for each class and pixel. @@ -15,11 +15,11 @@ format of ground truth: Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. -training , evaluation and output: +# Training , evaluation and output train and evaluation folder should have subfolder of images and labels. And output folder should be empty folder which the output model will be written there. -patches: +# Patches if you want to train your model with patches, the height and width of patches should be defined and also number of batchs (how many patches should be seen by model by each iteration). From cf18aa7fbb64900979b816b6b03ff20c5378b3a9 Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 16:13:37 +0100 Subject: [PATCH 008/374] Update README --- train/README | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/train/README b/train/README index e103b0b..5237d53 100644 --- a/train/README +++ b/train/README @@ -1,27 +1,2 @@ -# Train - just run: python train.py with config_params.json - - -# Ground truth format - - Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and - labels should be 0 and 1 for each class and pixel. - In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels - by pixels set from 0 , 1 ,2 .., n_classes-1. - The labels format should be png. - - If you have an image label for binary case it should look like this: - - Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] - this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. - -# Training , evaluation and output - train and evaluation folder should have subfolder of images and labels. - And output folder should be empty folder which the output model will be written there. - -# Patches - - if you want to train your model with patches, the height and width of patches should be defined and also number of - batchs (how many patches should be seen by model by each iteration). - In the case that model should see the image once, like page extraction, the patches should be set to false. + From ac542665815bea97752440bcf874a21ec939c047 Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 16:13:40 +0100 Subject: [PATCH 009/374] Delete README --- train/README | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 train/README diff --git a/train/README b/train/README deleted file mode 100644 index 5237d53..0000000 --- a/train/README +++ /dev/null @@ -1,2 +0,0 @@ - - From 350378af168d68f4709c1b98bc8e867e9b46ccfd Mon Sep 17 00:00:00 2001 From: "Rezanezhad, Vahid" Date: Thu, 5 Dec 2019 16:14:00 +0100 Subject: [PATCH 010/374] Add new file --- train/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 train/README.md diff --git a/train/README.md b/train/README.md new file mode 100644 index 0000000..c4dc27e --- /dev/null +++ b/train/README.md @@ -0,0 +1,26 @@ +# Train + just run: python train.py with config_params.json + + +# Ground truth format + + Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and + labels should be 0 and 1 for each class and pixel. + In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels + by pixels set from 0 , 1 ,2 .., n_classes-1. + The labels format should be png. + + If you have an image label for binary case it should look like this: + + Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] + this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. + +# Training , evaluation and output + train and evaluation folder should have subfolder of images and labels. + And output folder should be empty folder which the output model will be written there. + +# Patches + + if you want to train your model with patches, the height and width of patches should be defined and also number of + batchs (how many patches should be seen by model by each iteration). + In the case that model should see the image once, like page extraction, the patches should be set to false. \ No newline at end of file From 979b824aa8fe84619e9863372b45647ed8306327 Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Mon, 9 Dec 2019 15:33:53 +0100 Subject: [PATCH 011/374] =?UTF-8?q?=F0=9F=93=9D=20howto:=20Be=20more=20ver?= =?UTF-8?q?bose=20with=20the=20subtree=20pull?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- train/.gitkeep | 0 train/README.md | 26 +++ train/__init__.py | 0 train/config_params.json | 24 +++ train/metrics.py | 338 +++++++++++++++++++++++++++++++++++++++ train/models.py | 317 ++++++++++++++++++++++++++++++++++++ train/train.py | 192 ++++++++++++++++++++++ train/utils.py | 336 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 1233 insertions(+) create mode 100644 train/.gitkeep create mode 100644 train/README.md create mode 100644 train/__init__.py create mode 100644 train/config_params.json create mode 100644 train/metrics.py create mode 100644 train/models.py create mode 100644 train/train.py create mode 100644 train/utils.py diff --git a/train/.gitkeep b/train/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/train/README.md b/train/README.md new file mode 100644 index 0000000..c4dc27e --- /dev/null +++ b/train/README.md @@ -0,0 +1,26 @@ +# Train + just run: python train.py with config_params.json + + +# Ground truth format + + Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and + labels should be 0 and 1 for each class and pixel. + In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels + by pixels set from 0 , 1 ,2 .., n_classes-1. + The labels format should be png. + + If you have an image label for binary case it should look like this: + + Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] + this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. + +# Training , evaluation and output + train and evaluation folder should have subfolder of images and labels. + And output folder should be empty folder which the output model will be written there. + +# Patches + + if you want to train your model with patches, the height and width of patches should be defined and also number of + batchs (how many patches should be seen by model by each iteration). + In the case that model should see the image once, like page extraction, the patches should be set to false. \ No newline at end of file diff --git a/train/__init__.py b/train/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/train/config_params.json b/train/config_params.json new file mode 100644 index 0000000..5066444 --- /dev/null +++ b/train/config_params.json @@ -0,0 +1,24 @@ +{ + "n_classes" : 2, + "n_epochs" : 2, + "input_height" : 448, + "input_width" : 896, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : false, + "flip_aug" : false, + "elastic_aug" : false, + "blur_aug" : false, + "scaling" : false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "rotation": false, + "weighted_loss": true, + "dir_train": "../train", + "dir_eval": "../eval", + "dir_output": "../output" +} diff --git a/train/metrics.py b/train/metrics.py new file mode 100644 index 0000000..c63cc22 --- /dev/null +++ b/train/metrics.py @@ -0,0 +1,338 @@ +from keras import backend as K +import tensorflow as tf +import numpy as np + +def focal_loss(gamma=2., alpha=4.): + + gamma = float(gamma) + alpha = float(alpha) + + def focal_loss_fixed(y_true, y_pred): + """Focal loss for multi-classification + FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t) + Notice: y_pred is probability after softmax + gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper + d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x) + Focal Loss for Dense Object Detection + https://arxiv.org/abs/1708.02002 + + Arguments: + y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls] + y_pred {tensor} -- model's output, shape of [batch_size, num_cls] + + Keyword Arguments: + gamma {float} -- (default: {2.0}) + alpha {float} -- (default: {4.0}) + + Returns: + [tensor] -- loss. + """ + epsilon = 1.e-9 + y_true = tf.convert_to_tensor(y_true, tf.float32) + y_pred = tf.convert_to_tensor(y_pred, tf.float32) + + model_out = tf.add(y_pred, epsilon) + ce = tf.multiply(y_true, -tf.log(model_out)) + weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma)) + fl = tf.multiply(alpha, tf.multiply(weight, ce)) + reduced_fl = tf.reduce_max(fl, axis=1) + return tf.reduce_mean(reduced_fl) + return focal_loss_fixed + +def weighted_categorical_crossentropy(weights=None): + """ weighted_categorical_crossentropy + + Args: + * weights: crossentropy weights + Returns: + * weighted categorical crossentropy function + """ + + def loss(y_true, y_pred): + labels_floats = tf.cast(y_true, tf.float32) + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) + + if weights is not None: + weight_mask = tf.maximum(tf.reduce_max(tf.constant( + np.array(weights, dtype=np.float32)[None, None, None]) + * labels_floats, axis=-1), 1.0) + per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] + return tf.reduce_mean(per_pixel_loss) + return loss +def image_categorical_cross_entropy(y_true, y_pred, weights=None): + """ + :param y_true: tensor of shape (batch_size, height, width) representing the ground truth. + :param y_pred: tensor of shape (batch_size, height, width) representing the prediction. + :return: The mean cross-entropy on softmaxed tensors. + """ + + labels_floats = tf.cast(y_true, tf.float32) + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) + + if weights is not None: + weight_mask = tf.maximum( + tf.reduce_max(tf.constant( + np.array(weights, dtype=np.float32)[None, None, None]) + * labels_floats, axis=-1), 1.0) + per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] + + return tf.reduce_mean(per_pixel_loss) +def class_tversky(y_true, y_pred): + smooth = 1.0#1.00 + + y_true = K.permute_dimensions(y_true, (3,1,2,0)) + y_pred = K.permute_dimensions(y_pred, (3,1,2,0)) + + y_true_pos = K.batch_flatten(y_true) + y_pred_pos = K.batch_flatten(y_pred) + true_pos = K.sum(y_true_pos * y_pred_pos, 1) + false_neg = K.sum(y_true_pos * (1-y_pred_pos), 1) + false_pos = K.sum((1-y_true_pos)*y_pred_pos, 1) + alpha = 0.2#0.5 + beta=0.8 + return (true_pos + smooth)/(true_pos + alpha*false_neg + (beta)*false_pos + smooth) + +def focal_tversky_loss(y_true,y_pred): + pt_1 = class_tversky(y_true, y_pred) + gamma =1.3#4./3.0#1.3#4.0/3.00# 0.75 + return K.sum(K.pow((1-pt_1), gamma)) + +def generalized_dice_coeff2(y_true, y_pred): + n_el = 1 + for dim in y_true.shape: + n_el *= int(dim) + n_cl = y_true.shape[-1] + w = K.zeros(shape=(n_cl,)) + w = (K.sum(y_true, axis=(0,1,2)))/(n_el) + w = 1/(w**2+0.000001) + numerator = y_true*y_pred + numerator = w*K.sum(numerator,(0,1,2)) + numerator = K.sum(numerator) + denominator = y_true+y_pred + denominator = w*K.sum(denominator,(0,1,2)) + denominator = K.sum(denominator) + return 2*numerator/denominator +def generalized_dice_coeff(y_true, y_pred): + axes = tuple(range(1, len(y_pred.shape)-1)) + Ncl = y_pred.shape[-1] + w = K.zeros(shape=(Ncl,)) + w = K.sum(y_true, axis=axes) + w = 1/(w**2+0.000001) + # Compute gen dice coef: + numerator = y_true*y_pred + numerator = w*K.sum(numerator,axes) + numerator = K.sum(numerator) + + denominator = y_true+y_pred + denominator = w*K.sum(denominator,axes) + denominator = K.sum(denominator) + + gen_dice_coef = 2*numerator/denominator + + return gen_dice_coef + +def generalized_dice_loss(y_true, y_pred): + return 1 - generalized_dice_coeff2(y_true, y_pred) +def soft_dice_loss(y_true, y_pred, epsilon=1e-6): + ''' + Soft dice loss calculation for arbitrary batch size, number of classes, and number of spatial dimensions. + Assumes the `channels_last` format. + + # Arguments + y_true: b x X x Y( x Z...) x c One hot encoding of ground truth + y_pred: b x X x Y( x Z...) x c Network output, must sum to 1 over c channel (such as after softmax) + epsilon: Used for numerical stability to avoid divide by zero errors + + # References + V-Net: Fully Convolutional Neural Networks for Volumetric Medical Image Segmentation + https://arxiv.org/abs/1606.04797 + More details on Dice loss formulation + https://mediatum.ub.tum.de/doc/1395260/1395260.pdf (page 72) + + Adapted from https://github.com/Lasagne/Recipes/issues/99#issuecomment-347775022 + ''' + + # skip the batch and class axis for calculating Dice score + axes = tuple(range(1, len(y_pred.shape)-1)) + + numerator = 2. * K.sum(y_pred * y_true, axes) + + denominator = K.sum(K.square(y_pred) + K.square(y_true), axes) + return 1.00 - K.mean(numerator / (denominator + epsilon)) # average over classes and batch + +def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last = True, mean_per_class=False, verbose=False): + """ + Compute mean metrics of two segmentation masks, via Keras. + + IoU(A,B) = |A & B| / (| A U B|) + Dice(A,B) = 2*|A & B| / (|A| + |B|) + + Args: + y_true: true masks, one-hot encoded. + y_pred: predicted masks, either softmax outputs, or one-hot encoded. + metric_name: metric to be computed, either 'iou' or 'dice'. + metric_type: one of 'standard' (default), 'soft', 'naive'. + In the standard version, y_pred is one-hot encoded and the mean + is taken only over classes that are present (in y_true or y_pred). + The 'soft' version of the metrics are computed without one-hot + encoding y_pred. + The 'naive' version return mean metrics where absent classes contribute + to the class mean as 1.0 (instead of being dropped from the mean). + drop_last = True: boolean flag to drop last class (usually reserved + for background class in semantic segmentation) + mean_per_class = False: return mean along batch axis for each class. + verbose = False: print intermediate results such as intersection, union + (as number of pixels). + Returns: + IoU/Dice of y_true and y_pred, as a float, unless mean_per_class == True + in which case it returns the per-class metric, averaged over the batch. + + Inputs are B*W*H*N tensors, with + B = batch size, + W = width, + H = height, + N = number of classes + """ + + flag_soft = (metric_type == 'soft') + flag_naive_mean = (metric_type == 'naive') + + # always assume one or more classes + num_classes = K.shape(y_true)[-1] + + if not flag_soft: + # get one-hot encoded masks from y_pred (true masks should already be one-hot) + y_pred = K.one_hot(K.argmax(y_pred), num_classes) + y_true = K.one_hot(K.argmax(y_true), num_classes) + + # if already one-hot, could have skipped above command + # keras uses float32 instead of float64, would give error down (but numpy arrays or keras.to_categorical gives float64) + y_true = K.cast(y_true, 'float32') + y_pred = K.cast(y_pred, 'float32') + + # intersection and union shapes are batch_size * n_classes (values = area in pixels) + axes = (1,2) # W,H axes of each image + intersection = K.sum(K.abs(y_true * y_pred), axis=axes) + mask_sum = K.sum(K.abs(y_true), axis=axes) + K.sum(K.abs(y_pred), axis=axes) + union = mask_sum - intersection # or, np.logical_or(y_pred, y_true) for one-hot + + smooth = .001 + iou = (intersection + smooth) / (union + smooth) + dice = 2 * (intersection + smooth)/(mask_sum + smooth) + + metric = {'iou': iou, 'dice': dice}[metric_name] + + # define mask to be 0 when no pixels are present in either y_true or y_pred, 1 otherwise + mask = K.cast(K.not_equal(union, 0), 'float32') + + if drop_last: + metric = metric[:,:-1] + mask = mask[:,:-1] + + if verbose: + print('intersection, union') + print(K.eval(intersection), K.eval(union)) + print(K.eval(intersection/union)) + + # return mean metrics: remaining axes are (batch, classes) + if flag_naive_mean: + return K.mean(metric) + + # take mean only over non-absent classes + class_count = K.sum(mask, axis=0) + non_zero = tf.greater(class_count, 0) + non_zero_sum = tf.boolean_mask(K.sum(metric * mask, axis=0), non_zero) + non_zero_count = tf.boolean_mask(class_count, non_zero) + + if verbose: + print('Counts of inputs with class present, metrics for non-absent classes') + print(K.eval(class_count), K.eval(non_zero_sum / non_zero_count)) + + return K.mean(non_zero_sum / non_zero_count) + +def mean_iou(y_true, y_pred, **kwargs): + """ + Compute mean Intersection over Union of two segmentation masks, via Keras. + + Calls metrics_k(y_true, y_pred, metric_name='iou'), see there for allowed kwargs. + """ + return seg_metrics(y_true, y_pred, metric_name='iou', **kwargs) +def Mean_IOU(y_true, y_pred): + nb_classes = K.int_shape(y_pred)[-1] + iou = [] + true_pixels = K.argmax(y_true, axis=-1) + pred_pixels = K.argmax(y_pred, axis=-1) + void_labels = K.equal(K.sum(y_true, axis=-1), 0) + for i in range(0, nb_classes): # exclude first label (background) and last label (void) + true_labels = K.equal(true_pixels, i)# & ~void_labels + pred_labels = K.equal(pred_pixels, i)# & ~void_labels + inter = tf.to_int32(true_labels & pred_labels) + union = tf.to_int32(true_labels | pred_labels) + legal_batches = K.sum(tf.to_int32(true_labels), axis=1)>0 + ious = K.sum(inter, axis=1)/K.sum(union, axis=1) + iou.append(K.mean(tf.gather(ious, indices=tf.where(legal_batches)))) # returns average IoU of the same objects + iou = tf.stack(iou) + legal_labels = ~tf.debugging.is_nan(iou) + iou = tf.gather(iou, indices=tf.where(legal_labels)) + return K.mean(iou) + +def iou_vahid(y_true, y_pred): + nb_classes = tf.shape(y_true)[-1]+tf.to_int32(1) + true_pixels = K.argmax(y_true, axis=-1) + pred_pixels = K.argmax(y_pred, axis=-1) + iou = [] + + for i in tf.range(nb_classes): + tp=K.sum( tf.to_int32( K.equal(true_pixels, i) & K.equal(pred_pixels, i) ) ) + fp=K.sum( tf.to_int32( K.not_equal(true_pixels, i) & K.equal(pred_pixels, i) ) ) + fn=K.sum( tf.to_int32( K.equal(true_pixels, i) & K.not_equal(pred_pixels, i) ) ) + iouh=tp/(tp+fp+fn) + iou.append(iouh) + return K.mean(iou) + + +def IoU_metric(Yi,y_predi): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + y_predi = np.argmax(y_predi, axis=3) + y_testi = np.argmax(Yi, axis=3) + IoUs = [] + Nclass = int(np.max(Yi)) + 1 + for c in range(Nclass): + TP = np.sum( (Yi == c)&(y_predi==c) ) + FP = np.sum( (Yi != c)&(y_predi==c) ) + FN = np.sum( (Yi == c)&(y_predi != c)) + IoU = TP/float(TP + FP + FN) + IoUs.append(IoU) + return K.cast( np.mean(IoUs) ,dtype='float32' ) + + +def IoU_metric_keras(y_true, y_pred): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + init = tf.global_variables_initializer() + sess = tf.Session() + sess.run(init) + + return IoU_metric(y_true.eval(session=sess), y_pred.eval(session=sess)) + +def jaccard_distance_loss(y_true, y_pred, smooth=100): + """ + Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|) + = sum(|A*B|)/(sum(|A|)+sum(|B|)-sum(|A*B|)) + + The jaccard distance loss is usefull for unbalanced datasets. This has been + shifted so it converges on 0 and is smoothed to avoid exploding or disapearing + gradient. + + Ref: https://en.wikipedia.org/wiki/Jaccard_index + + @url: https://gist.github.com/wassname/f1452b748efcbeb4cb9b1d059dce6f96 + @author: wassname + """ + intersection = K.sum(K.abs(y_true * y_pred), axis=-1) + sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1) + jac = (intersection + smooth) / (sum_ - intersection + smooth) + return (1 - jac) * smooth + + diff --git a/train/models.py b/train/models.py new file mode 100644 index 0000000..7c806b4 --- /dev/null +++ b/train/models.py @@ -0,0 +1,317 @@ +from keras.models import * +from keras.layers import * +from keras import layers +from keras.regularizers import l2 + +resnet50_Weights_path='./pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' +IMAGE_ORDERING ='channels_last' +MERGE_AXIS=-1 + + +def one_side_pad( x ): + x = ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING)(x) + if IMAGE_ORDERING == 'channels_first': + x = Lambda(lambda x : x[: , : , :-1 , :-1 ] )(x) + elif IMAGE_ORDERING == 'channels_last': + x = Lambda(lambda x : x[: , :-1 , :-1 , : ] )(x) + return x + +def identity_block(input_tensor, kernel_size, filters, stage, block): + """The identity block is the block that has no conv layer at shortcut. + # Arguments + input_tensor: input tensor + kernel_size: defualt 3, the kernel size of middle conv layer at main path + filters: list of integers, the filterss of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + # Returns + Output tensor for the block. + """ + filters1, filters2, filters3 = filters + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = Conv2D(filters1, (1, 1) , data_format=IMAGE_ORDERING , name=conv_name_base + '2a')(input_tensor) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) + x = Activation('relu')(x) + + x = Conv2D(filters2, kernel_size , data_format=IMAGE_ORDERING , + padding='same', name=conv_name_base + '2b')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) + x = Activation('relu')(x) + + x = Conv2D(filters3 , (1, 1), data_format=IMAGE_ORDERING , name=conv_name_base + '2c')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) + + x = layers.add([x, input_tensor]) + x = Activation('relu')(x) + return x + + +def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)): + """conv_block is the block that has a conv layer at shortcut + # Arguments + input_tensor: input tensor + kernel_size: defualt 3, the kernel size of middle conv layer at main path + filters: list of integers, the filterss of 3 conv layer at main path + stage: integer, current stage label, used for generating layer names + block: 'a','b'..., current block label, used for generating layer names + # Returns + Output tensor for the block. + Note that from stage 3, the first conv layer at main path is with strides=(2,2) + And the shortcut should have strides=(2,2) as well + """ + filters1, filters2, filters3 = filters + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + conv_name_base = 'res' + str(stage) + block + '_branch' + bn_name_base = 'bn' + str(stage) + block + '_branch' + + x = Conv2D(filters1, (1, 1) , data_format=IMAGE_ORDERING , strides=strides, + name=conv_name_base + '2a')(input_tensor) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) + x = Activation('relu')(x) + + x = Conv2D(filters2, kernel_size , data_format=IMAGE_ORDERING , padding='same', + name=conv_name_base + '2b')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) + x = Activation('relu')(x) + + x = Conv2D(filters3, (1, 1) , data_format=IMAGE_ORDERING , name=conv_name_base + '2c')(x) + x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) + + shortcut = Conv2D(filters3, (1, 1) , data_format=IMAGE_ORDERING , strides=strides, + name=conv_name_base + '1')(input_tensor) + shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut) + + x = layers.add([x, shortcut]) + x = Activation('relu')(x) + return x + + +def resnet50_unet_light(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + assert input_height%32 == 0 + assert input_width%32 == 0 + + + img_input = Input(shape=(input_height,input_width , 3 )) + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) + + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x ) + + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + + if pretraining: + model=Model( img_input , x ).load_weights(resnet50_Weights_path) + + + v512_2048 = Conv2D( 512 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f5 ) + v512_2048 = ( BatchNormalization(axis=bn_axis))(v512_2048) + v512_2048 = Activation('relu')(v512_2048) + + + + v512_1024=Conv2D( 512 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f4 ) + v512_1024 = ( BatchNormalization(axis=bn_axis))(v512_1024) + v512_1024 = Activation('relu')(v512_1024) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(v512_2048) + o = ( concatenate([ o ,v512_1024],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([ o ,f3],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f2],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 128 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay) ) )(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f1],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 64 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,img_input],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 32 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + + o = Conv2D( n_classes , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( o ) + o = ( BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + + + model = Model( img_input , o ) + return model + +def resnet50_unet(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + assert input_height%32 == 0 + assert input_width%32 == 0 + + + img_input = Input(shape=(input_height,input_width , 3 )) + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) + + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x ) + + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + if pretraining: + Model( img_input , x ).load_weights(resnet50_Weights_path) + + v1024_2048 = Conv2D( 1024 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f5 ) + v1024_2048 = ( BatchNormalization(axis=bn_axis))(v1024_2048) + v1024_2048 = Activation('relu')(v1024_2048) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(v1024_2048) + o = ( concatenate([ o ,f4],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([ o ,f3],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f2],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING))(o) + o = ( Conv2D( 128 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay) ) )(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,f1],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 64 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) + o = ( concatenate([o,img_input],axis=MERGE_AXIS ) ) + o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) + o = ( Conv2D( 32 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) + o = ( BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + + o = Conv2D( n_classes , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( o ) + o = ( BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + + model = Model( img_input , o ) + + + + + return model diff --git a/train/train.py b/train/train.py new file mode 100644 index 0000000..07c7418 --- /dev/null +++ b/train/train.py @@ -0,0 +1,192 @@ +import os +import sys +import tensorflow as tf +from keras.backend.tensorflow_backend import set_session +import keras , warnings +from keras.optimizers import * +from sacred import Experiment +from models import * +from utils import * +from metrics import * + + +def configuration(): + keras.backend.clear_session() + tf.reset_default_graph() + warnings.filterwarnings('ignore') + + os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' + config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) + + + config.gpu_options.allow_growth = True + config.gpu_options.per_process_gpu_memory_fraction=0.95#0.95 + config.gpu_options.visible_device_list="0" + set_session(tf.Session(config=config)) + +def get_dirs_or_files(input_data): + if os.path.isdir(input_data): + image_input, labels_input = os.path.join(input_data, 'images/'), os.path.join(input_data, 'labels/') + # Check if training dir exists + assert os.path.isdir(image_input), "{} is not a directory".format(image_input) + assert os.path.isdir(labels_input), "{} is not a directory".format(labels_input) + return image_input, labels_input + +ex = Experiment() + +@ex.config +def config_params(): + n_classes=None # Number of classes. If your case study is binary case the set it to 2 and otherwise give your number of cases. + n_epochs=1 + input_height=224*1 + input_width=224*1 + weight_decay=1e-6 # Weight decay of l2 regularization of model layers. + n_batch=1 # Number of batches at each iteration. + learning_rate=1e-4 + patches=False # Make patches of image in order to use all information of image. In the case of page + # extraction this should be set to false since model should see all image. + augmentation=False + flip_aug=False # Flip image (augmentation). + elastic_aug=False # Elastic transformation (augmentation). + blur_aug=False # Blur patches of image (augmentation). + scaling=False # Scaling of patches (augmentation) will be imposed if this set to true. + binarization=False # Otsu thresholding. Used for augmentation in the case of binary case like textline prediction. For multicases should not be applied. + dir_train=None # Directory of training dataset (sub-folders should be named images and labels). + dir_eval=None # Directory of validation dataset (sub-folders should be named images and labels). + dir_output=None # Directory of output where the model should be saved. + pretraining=False # Set true to load pretrained weights of resnet50 encoder. + weighted_loss=False # Set True if classes are unbalanced and you want to use weighted loss function. + scaling_bluring=False + rotation: False + scaling_binarization=False + blur_k=['blur','guass','median'] # Used in order to blur image. Used for augmentation. + scales=[0.9 , 1.1 ] # Scale patches with these scales. Used for augmentation. + flip_index=[0,1] # Flip image. Used for augmentation. + + +@ex.automain +def run(n_classes,n_epochs,input_height, + input_width,weight_decay,weighted_loss, + n_batch,patches,augmentation,flip_aug,blur_aug,scaling, binarization, + blur_k,scales,dir_train, + scaling_bluring,scaling_binarization,rotation, + flip_index,dir_eval ,dir_output,pretraining,learning_rate): + + dir_img,dir_seg=get_dirs_or_files(dir_train) + dir_img_val,dir_seg_val=get_dirs_or_files(dir_eval) + + # make first a directory in output for both training and evaluations in order to flow data from these directories. + dir_train_flowing=os.path.join(dir_output,'train') + dir_eval_flowing=os.path.join(dir_output,'eval') + + dir_flow_train_imgs=os.path.join(dir_train_flowing,'images') + dir_flow_train_labels=os.path.join(dir_train_flowing,'labels') + + dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images') + dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels') + + if os.path.isdir(dir_train_flowing): + os.system('rm -rf '+dir_train_flowing) + os.makedirs(dir_train_flowing) + else: + os.makedirs(dir_train_flowing) + + if os.path.isdir(dir_eval_flowing): + os.system('rm -rf '+dir_eval_flowing) + os.makedirs(dir_eval_flowing) + else: + os.makedirs(dir_eval_flowing) + + + os.mkdir(dir_flow_train_imgs) + os.mkdir(dir_flow_train_labels) + + os.mkdir(dir_flow_eval_imgs) + os.mkdir(dir_flow_eval_labels) + + + + #set the gpu configuration + configuration() + + + #writing patches into a sub-folder in order to be flowed from directory. + provide_patches(dir_img,dir_seg,dir_flow_train_imgs, + dir_flow_train_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + augmentation=augmentation,patches=patches) + + provide_patches(dir_img_val,dir_seg_val,dir_flow_eval_imgs, + dir_flow_eval_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + augmentation=False,patches=patches) + + if weighted_loss: + weights=np.zeros(n_classes) + for obj in os.listdir(dir_seg): + label_obj=cv2.imread(dir_seg+'/'+obj) + label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) + weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + + + weights=1.00/weights + + weights=weights/float(np.sum(weights)) + weights=weights/float(np.min(weights)) + weights=weights/float(np.sum(weights)) + + + + + #get our model. + model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + + #if you want to see the model structure just uncomment model summary. + #model.summary() + + + if not weighted_loss: + model.compile(loss='categorical_crossentropy', + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + if weighted_loss: + model.compile(loss=weighted_categorical_crossentropy(weights), + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + + mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', + save_weights_only=True, period=1) + + + #generating train and evaluation data + train_gen = data_gen(dir_flow_train_imgs,dir_flow_train_labels, batch_size = n_batch, + input_height=input_height, input_width=input_width,n_classes=n_classes ) + val_gen = data_gen(dir_flow_eval_imgs,dir_flow_eval_labels, batch_size = n_batch, + input_height=input_height, input_width=input_width,n_classes=n_classes ) + + + model.fit_generator( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch), + validation_data=val_gen, + validation_steps=1, + epochs=n_epochs) + + + + os.system('rm -rf '+dir_train_flowing) + os.system('rm -rf '+dir_eval_flowing) + + model.save(dir_output+'/'+'model'+'.h5') + + + + + + + + + + diff --git a/train/utils.py b/train/utils.py new file mode 100644 index 0000000..afdc9e5 --- /dev/null +++ b/train/utils.py @@ -0,0 +1,336 @@ +import os +import cv2 +import numpy as np +import seaborn as sns +from scipy.ndimage.interpolation import map_coordinates +from scipy.ndimage.filters import gaussian_filter +import random +from tqdm import tqdm + + + + +def bluring(img_in,kind): + if kind=='guass': + img_blur = cv2.GaussianBlur(img_in,(5,5),0) + elif kind=="median": + img_blur = cv2.medianBlur(img_in,5) + elif kind=='blur': + img_blur=cv2.blur(img_in,(5,5)) + return img_blur + +def color_images(seg, n_classes): + ann_u=range(n_classes) + if len(np.shape(seg))==3: + seg=seg[:,:,0] + + seg_img=np.zeros((np.shape(seg)[0],np.shape(seg)[1],3)).astype(float) + colors=sns.color_palette("hls", n_classes) + + for c in ann_u: + c=int(c) + segl=(seg==c) + seg_img[:,:,0]+=segl*(colors[c][0]) + seg_img[:,:,1]+=segl*(colors[c][1]) + seg_img[:,:,2]+=segl*(colors[c][2]) + return seg_img + + +def resize_image(seg_in,input_height,input_width): + return cv2.resize(seg_in,(input_width,input_height),interpolation=cv2.INTER_NEAREST) +def get_one_hot(seg,input_height,input_width,n_classes): + seg=seg[:,:,0] + seg_f=np.zeros((input_height, input_width,n_classes)) + for j in range(n_classes): + seg_f[:,:,j]=(seg==j).astype(int) + return seg_f + + +def IoU(Yi,y_predi): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + + IoUs = [] + classes_true=np.unique(Yi) + for c in classes_true: + TP = np.sum( (Yi == c)&(y_predi==c) ) + FP = np.sum( (Yi != c)&(y_predi==c) ) + FN = np.sum( (Yi == c)&(y_predi != c)) + IoU = TP/float(TP + FP + FN) + print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c,TP,FP,FN,IoU)) + IoUs.append(IoU) + mIoU = np.mean(IoUs) + print("_________________") + print("Mean IoU: {:4.3f}".format(mIoU)) + return mIoU +def data_gen(img_folder, mask_folder, batch_size,input_height, input_width,n_classes): + c = 0 + n = os.listdir(img_folder) #List of training images + random.shuffle(n) + while True: + img = np.zeros((batch_size, input_height, input_width, 3)).astype('float') + mask = np.zeros((batch_size, input_height, input_width, n_classes)).astype('float') + + for i in range(c, c+batch_size): #initially from 0 to 16, c = 0. + #print(img_folder+'/'+n[i]) + filename=n[i].split('.')[0] + train_img = cv2.imread(img_folder+'/'+n[i])/255. + train_img = cv2.resize(train_img, (input_width, input_height),interpolation=cv2.INTER_NEAREST)# Read an image from folder and resize + + img[i-c] = train_img #add to array - img[0], img[1], and so on. + train_mask = cv2.imread(mask_folder+'/'+filename+'.png') + #print(mask_folder+'/'+filename+'.png') + #print(train_mask.shape) + train_mask = get_one_hot( resize_image(train_mask,input_height,input_width),input_height,input_width,n_classes) + #train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] + + mask[i-c] = train_mask + + c+=batch_size + if(c+batch_size>=len(os.listdir(img_folder))): + c=0 + random.shuffle(n) + yield img, mask + +def otsu_copy(img): + img_r=np.zeros(img.shape) + img1=img[:,:,0] + img2=img[:,:,1] + img3=img[:,:,2] + _, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold2 = cv2.threshold(img2, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold3 = cv2.threshold(img3, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + img_r[:,:,0]=threshold1 + img_r[:,:,1]=threshold1 + img_r[:,:,2]=threshold1 + return img_r + +def rotation_90(img): + img_rot=np.zeros((img.shape[1],img.shape[0],img.shape[2])) + img_rot[:,:,0]=img[:,:,0].T + img_rot[:,:,1]=img[:,:,1].T + img_rot[:,:,2]=img[:,:,2].T + return img_rot + +def get_patches(dir_img_f,dir_seg_f,img,label,height,width,indexer): + + + img_h=img.shape[0] + img_w=img.shape[1] + + nxf=img_w/float(width) + nyf=img_h/float(height) + + if nxf>int(nxf): + nxf=int(nxf)+1 + if nyf>int(nyf): + nyf=int(nyf)+1 + + nxf=int(nxf) + nyf=int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d=i*width + index_x_u=(i+1)*width + + index_y_d=j*height + index_y_u=(j+1)*height + + if index_x_u>img_w: + index_x_u=img_w + index_x_d=img_w-width + if index_y_u>img_h: + index_y_u=img_h + index_y_d=img_h-height + + + img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] + label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] + + cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) + cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) + indexer+=1 + return indexer + + + +def get_patches_num_scale(dir_img_f,dir_seg_f,img,label,height,width,indexer,scaler): + + + img_h=img.shape[0] + img_w=img.shape[1] + + height_scale=int(height*scaler) + width_scale=int(width*scaler) + + + nxf=img_w/float(width_scale) + nyf=img_h/float(height_scale) + + if nxf>int(nxf): + nxf=int(nxf)+1 + if nyf>int(nyf): + nyf=int(nyf)+1 + + nxf=int(nxf) + nyf=int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d=i*width_scale + index_x_u=(i+1)*width_scale + + index_y_d=j*height_scale + index_y_u=(j+1)*height_scale + + if index_x_u>img_w: + index_x_u=img_w + index_x_d=img_w-width_scale + if index_y_u>img_h: + index_y_u=img_h + index_y_d=img_h-height_scale + + + img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] + label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] + + img_patch=resize_image(img_patch,height,width) + label_patch=resize_image(label_patch,height,width) + + cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) + cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) + indexer+=1 + + return indexer + + + +def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, + dir_flow_train_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + augmentation=False,patches=False): + + imgs_cv_train=np.array(os.listdir(dir_img)) + segs_cv_train=np.array(os.listdir(dir_seg)) + + indexer=0 + for im, seg_i in tqdm(zip(imgs_cv_train,segs_cv_train)): + img_name=im.split('.')[0] + + if not patches: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', resize_image(cv2.imread(dir_img+'/'+im),input_height,input_width ) ) + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width ) ) + indexer+=1 + + if augmentation: + if rotation: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + rotation_90( resize_image(cv2.imread(dir_img+'/'+im), + input_height,input_width) ) ) + + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png', + rotation_90 ( resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width) ) ) + indexer+=1 + + if flip_aug: + for f_i in flip_index: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + resize_image(cv2.flip(cv2.imread(dir_img+'/'+im),f_i),input_height,input_width) ) + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , + resize_image(cv2.flip(cv2.imread(dir_seg+'/'+img_name+'.png'),f_i),input_height,input_width) ) + indexer+=1 + + if blur_aug: + for blur_i in blur_k: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + (resize_image(bluring(cv2.imread(dir_img+'/'+im),blur_i),input_height,input_width) ) ) + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , + resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width) ) + indexer+=1 + + + if binarization: + cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', + resize_image(otsu_copy( cv2.imread(dir_img+'/'+im)),input_height,input_width )) + + cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png', + resize_image( cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width )) + indexer+=1 + + + + + + + if patches: + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) + + if augmentation: + + if rotation: + + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + rotation_90( cv2.imread(dir_img+'/'+im) ), + rotation_90( cv2.imread(dir_seg+'/'+img_name+'.png') ), + input_height,input_width,indexer=indexer) + if flip_aug: + for f_i in flip_index: + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + cv2.flip( cv2.imread(dir_img+'/'+im) , f_i), + cv2.flip( cv2.imread(dir_seg+'/'+img_name+'.png') ,f_i), + input_height,input_width,indexer=indexer) + if blur_aug: + for blur_i in blur_k: + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + bluring( cv2.imread(dir_img+'/'+im) , blur_i), + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) + + + if scaling: + for sc_ind in scales: + indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + cv2.imread(dir_img+'/'+im) , + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer,scaler=sc_ind) + if binarization: + + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + otsu_copy( cv2.imread(dir_img+'/'+im)), + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) + + + + if scaling_bluring: + for sc_ind in scales: + for blur_i in blur_k: + indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + bluring( cv2.imread(dir_img+'/'+im) , blur_i) , + cv2.imread(dir_seg+'/'+img_name+'.png') , + input_height,input_width,indexer=indexer,scaler=sc_ind) + + if scaling_binarization: + for sc_ind in scales: + indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + otsu_copy( cv2.imread(dir_img+'/'+im)) , + cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer,scaler=sc_ind) + + + + + + From 8084e136ba67513caa4e5309be70caff2b75fbea Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 10 Dec 2019 11:57:37 +0100 Subject: [PATCH 012/374] Update README --- train/README.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/train/README.md b/train/README.md index c4dc27e..16e5dce 100644 --- a/train/README.md +++ b/train/README.md @@ -4,16 +4,21 @@ # Ground truth format - Lables for each pixel is identified by a number . So if you have a binary case n_classes should be set to 2 and + Lables for each pixel is identified by a number . So if you have a + binary case n_classes should be set to 2 and labels should be 0 and 1 for each class and pixel. - In the case of multiclass just set n_classes to the number of classes you have and the try to produce the labels + In the case of multiclass just set n_classes to the number of classes + you have and the try to produce the labels by pixels set from 0 , 1 ,2 .., n_classes-1. The labels format should be png. If you have an image label for binary case it should look like this: - Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ,[[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] - this means that you have an image by 3*4*3 and pixel[0,0] belongs to class 1 and pixel[0,1] to class 0. + Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], + [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] , + [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] + This means that you have an image by 3*4*3 and pixel[0,0] belongs + to class 1 and pixel[0,1] to class 0. # Training , evaluation and output train and evaluation folder should have subfolder of images and labels. @@ -21,6 +26,11 @@ # Patches - if you want to train your model with patches, the height and width of patches should be defined and also number of + if you want to train your model with patches, the height and width of + patches should be defined and also number of batchs (how many patches should be seen by model by each iteration). - In the case that model should see the image once, like page extraction, the patches should be set to false. \ No newline at end of file + In the case that model should see the image once, like page extraction, + the patches should be set to false. +# Pretrained encoder +Download weights from this limk and add it to pretrained_model folder. +https://file.spk-berlin.de:8443/pretrained_encoder/ From 4229ad92d7460ed9fdc63a2837527586fde18de3 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 10 Dec 2019 11:58:02 +0100 Subject: [PATCH 013/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 16e5dce..3ba90a1 100644 --- a/train/README.md +++ b/train/README.md @@ -32,5 +32,5 @@ In the case that model should see the image once, like page extraction, the patches should be set to false. # Pretrained encoder -Download weights from this limk and add it to pretrained_model folder. +Download weights from this link and add it to pretrained_model folder. https://file.spk-berlin.de:8443/pretrained_encoder/ From b5f9b9c54ad4ad746ab93bc7f81652f9158d75e5 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 10 Dec 2019 14:01:55 +0100 Subject: [PATCH 014/374] Update main.py --- train/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/train.py b/train/train.py index 07c7418..baeb847 100644 --- a/train/train.py +++ b/train/train.py @@ -169,7 +169,7 @@ def run(n_classes,n_epochs,input_height, model.fit_generator( train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch), + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch)-1, validation_data=val_gen, validation_steps=1, epochs=n_epochs) From df536d62c04825e05ea5aceb6067616db3b357a8 Mon Sep 17 00:00:00 2001 From: Clemens Neudecker <952378+cneud@users.noreply.github.com> Date: Tue, 10 Dec 2019 16:39:41 +0100 Subject: [PATCH 015/374] Add LICENSE --- train/LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 train/LICENSE diff --git a/train/LICENSE b/train/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/train/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From ad1360b179e0f4c39882bdd119e1760c7747db4d Mon Sep 17 00:00:00 2001 From: Clemens Neudecker <952378+cneud@users.noreply.github.com> Date: Wed, 15 Jan 2020 19:37:27 +0100 Subject: [PATCH 016/374] Update README.md --- train/README.md | 65 +++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/train/README.md b/train/README.md index 3ba90a1..4c49f39 100644 --- a/train/README.md +++ b/train/README.md @@ -1,36 +1,47 @@ -# Train - just run: python train.py with config_params.json +# Pixelwise Segmentation +> Pixelwise segmentation for document images + +## Introduction +This repository contains the source code for training an encoder model for document image segmentation. + +## Installation +Either clone the repository via `git clone https://github.com/qurator-spk/sbb_pixelwise_segmentation.git` or download and unpack the [ZIP](https://github.com/qurator-spk/sbb_pixelwise_segmentation/archive/master.zip). + +## Usage + +### Train +To train a model, run: ``python train.py with config_params.json`` + +### Ground truth format +Lables for each pixel are identified by a number. So if you have a +binary case, ``n_classes`` should be set to ``2`` and labels should +be ``0`` and ``1`` for each class and pixel. + +In the case of multiclass, just set ``n_classes`` to the number of classes +you have and the try to produce the labels by pixels set from ``0 , 1 ,2 .., n_classes-1``. +The labels format should be png. - -# Ground truth format - - Lables for each pixel is identified by a number . So if you have a - binary case n_classes should be set to 2 and - labels should be 0 and 1 for each class and pixel. - In the case of multiclass just set n_classes to the number of classes - you have and the try to produce the labels - by pixels set from 0 , 1 ,2 .., n_classes-1. - The labels format should be png. - - If you have an image label for binary case it should look like this: +If you have an image label for a binary case it should look like this: Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] , [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] - This means that you have an image by 3*4*3 and pixel[0,0] belongs - to class 1 and pixel[0,1] to class 0. -# Training , evaluation and output - train and evaluation folder should have subfolder of images and labels. - And output folder should be empty folder which the output model will be written there. + This means that you have an image by `3*4*3` and `pixel[0,0]` belongs + to class `1` and `pixel[0,1]` belongs to class `0`. + +### Training , evaluation and output +The train and evaluation folders should contain subfolders of images and labels. +The output folder should be an empty folder where the output model will be written to. # Patches +If you want to train your model with patches, the height and width of +the patches should be defined and also the number of batches (how many patches +should be seen by the model in each iteration). + +In the case that the model should see the image once, like page extraction, +patches should be set to ``false``. - if you want to train your model with patches, the height and width of - patches should be defined and also number of - batchs (how many patches should be seen by model by each iteration). - In the case that model should see the image once, like page extraction, - the patches should be set to false. -# Pretrained encoder -Download weights from this link and add it to pretrained_model folder. -https://file.spk-berlin.de:8443/pretrained_encoder/ +### Pretrained encoder +Download our pretrained weights and add them to a ``pretrained_model`` folder: +~~https://file.spk-berlin.de:8443/pretrained_encoder/~~ From 66d7138343edc9fe3d7d918198a1f20b4112e42b Mon Sep 17 00:00:00 2001 From: Clemens Neudecker <952378+cneud@users.noreply.github.com> Date: Wed, 15 Jan 2020 19:43:31 +0100 Subject: [PATCH 017/374] Update README.md --- train/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/train/README.md b/train/README.md index 4c49f39..18495a5 100644 --- a/train/README.md +++ b/train/README.md @@ -7,6 +7,9 @@ This repository contains the source code for training an encoder model for docum ## Installation Either clone the repository via `git clone https://github.com/qurator-spk/sbb_pixelwise_segmentation.git` or download and unpack the [ZIP](https://github.com/qurator-spk/sbb_pixelwise_segmentation/archive/master.zip). +### Pretrained encoder +Download our pretrained weights and add them to a ``pretrained_model`` folder: +~~https://file.spk-berlin.de:8443/pretrained_encoder/~~ ## Usage ### Train @@ -34,7 +37,7 @@ If you have an image label for a binary case it should look like this: The train and evaluation folders should contain subfolders of images and labels. The output folder should be an empty folder where the output model will be written to. -# Patches +### Patches If you want to train your model with patches, the height and width of the patches should be defined and also the number of batches (how many patches should be seen by the model in each iteration). @@ -42,6 +45,4 @@ should be seen by the model in each iteration). In the case that the model should see the image once, like page extraction, patches should be set to ``false``. -### Pretrained encoder -Download our pretrained weights and add them to a ``pretrained_model`` folder: -~~https://file.spk-berlin.de:8443/pretrained_encoder/~~ + From 4e216475dca544515488071f035cde639d053584 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 16 Jan 2020 15:53:39 +0100 Subject: [PATCH 018/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 18495a5..ede05dd 100644 --- a/train/README.md +++ b/train/README.md @@ -9,7 +9,7 @@ Either clone the repository via `git clone https://github.com/qurator-spk/sbb_pi ### Pretrained encoder Download our pretrained weights and add them to a ``pretrained_model`` folder: -~~https://file.spk-berlin.de:8443/pretrained_encoder/~~ +https://qurator-data.de/pretrained_encoder/ ## Usage ### Train From b54285b19684e6a6b86a52448dc9afd4a38e95ea Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 16 Jan 2020 16:05:06 +0100 Subject: [PATCH 019/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index ede05dd..d0d26d6 100644 --- a/train/README.md +++ b/train/README.md @@ -9,7 +9,7 @@ Either clone the repository via `git clone https://github.com/qurator-spk/sbb_pi ### Pretrained encoder Download our pretrained weights and add them to a ``pretrained_model`` folder: -https://qurator-data.de/pretrained_encoder/ +https://qurator-data.de/sbb_pixelwise_segmentation/pretrained_encoder/ ## Usage ### Train From 070c2e046259441b712d11be21eb26c6db191b71 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 22 Jun 2021 14:20:51 -0400 Subject: [PATCH 020/374] first updates, padding, rotations --- train/config_params.json | 22 ++-- train/train.py | 183 ++++++++++++++------------- train/utils.py | 265 +++++++++++++++++++++++++++++++-------- 3 files changed, 319 insertions(+), 151 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index 5066444..d8f1ac5 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,24 +1,24 @@ { - "n_classes" : 2, - "n_epochs" : 2, + "n_classes" : 3, + "n_epochs" : 1, "input_height" : 448, - "input_width" : 896, + "input_width" : 672, "weight_decay" : 1e-6, - "n_batch" : 1, + "n_batch" : 2, "learning_rate": 1e-4, "patches" : true, "pretraining" : true, - "augmentation" : false, + "augmentation" : true, "flip_aug" : false, - "elastic_aug" : false, - "blur_aug" : false, + "blur_aug" : true, "scaling" : false, "binarization" : false, "scaling_bluring" : false, "scaling_binarization" : false, + "scaling_flip" : false, "rotation": false, - "weighted_loss": true, - "dir_train": "../train", - "dir_eval": "../eval", - "dir_output": "../output" + "rotation_not_90": false, + "dir_train": "/home/vahid/Documents/handwrittens_train/train", + "dir_eval": "/home/vahid/Documents/handwrittens_train/eval", + "dir_output": "/home/vahid/Documents/handwrittens_train/output" } diff --git a/train/train.py b/train/train.py index baeb847..c256d83 100644 --- a/train/train.py +++ b/train/train.py @@ -8,7 +8,7 @@ from sacred import Experiment from models import * from utils import * from metrics import * - +from keras.models import load_model def configuration(): keras.backend.clear_session() @@ -47,7 +47,6 @@ def config_params(): # extraction this should be set to false since model should see all image. augmentation=False flip_aug=False # Flip image (augmentation). - elastic_aug=False # Elastic transformation (augmentation). blur_aug=False # Blur patches of image (augmentation). scaling=False # Scaling of patches (augmentation) will be imposed if this set to true. binarization=False # Otsu thresholding. Used for augmentation in the case of binary case like textline prediction. For multicases should not be applied. @@ -55,110 +54,116 @@ def config_params(): dir_eval=None # Directory of validation dataset (sub-folders should be named images and labels). dir_output=None # Directory of output where the model should be saved. pretraining=False # Set true to load pretrained weights of resnet50 encoder. - weighted_loss=False # Set True if classes are unbalanced and you want to use weighted loss function. scaling_bluring=False - rotation: False scaling_binarization=False + scaling_flip=False + thetha=[10,-10] blur_k=['blur','guass','median'] # Used in order to blur image. Used for augmentation. - scales=[0.9 , 1.1 ] # Scale patches with these scales. Used for augmentation. - flip_index=[0,1] # Flip image. Used for augmentation. + scales= [ 0.5, 2 ] # Scale patches with these scales. Used for augmentation. + flip_index=[0,1,-1] # Flip image. Used for augmentation. @ex.automain def run(n_classes,n_epochs,input_height, - input_width,weight_decay,weighted_loss, - n_batch,patches,augmentation,flip_aug,blur_aug,scaling, binarization, + input_width,weight_decay, + n_batch,patches,augmentation,flip_aug + ,blur_aug,scaling, binarization, blur_k,scales,dir_train, scaling_bluring,scaling_binarization,rotation, + rotation_not_90,thetha,scaling_flip, flip_index,dir_eval ,dir_output,pretraining,learning_rate): - dir_img,dir_seg=get_dirs_or_files(dir_train) - dir_img_val,dir_seg_val=get_dirs_or_files(dir_eval) + data_is_provided = False - # make first a directory in output for both training and evaluations in order to flow data from these directories. - dir_train_flowing=os.path.join(dir_output,'train') - dir_eval_flowing=os.path.join(dir_output,'eval') - - dir_flow_train_imgs=os.path.join(dir_train_flowing,'images') - dir_flow_train_labels=os.path.join(dir_train_flowing,'labels') - - dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images') - dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels') - - if os.path.isdir(dir_train_flowing): - os.system('rm -rf '+dir_train_flowing) - os.makedirs(dir_train_flowing) + if data_is_provided: + dir_train_flowing=os.path.join(dir_output,'train') + dir_eval_flowing=os.path.join(dir_output,'eval') + + dir_flow_train_imgs=os.path.join(dir_train_flowing,'images') + dir_flow_train_labels=os.path.join(dir_train_flowing,'labels') + + dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images') + dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels') + + configuration() + else: - os.makedirs(dir_train_flowing) + dir_img,dir_seg=get_dirs_or_files(dir_train) + dir_img_val,dir_seg_val=get_dirs_or_files(dir_eval) - if os.path.isdir(dir_eval_flowing): - os.system('rm -rf '+dir_eval_flowing) - os.makedirs(dir_eval_flowing) - else: - os.makedirs(dir_eval_flowing) + # make first a directory in output for both training and evaluations in order to flow data from these directories. + dir_train_flowing=os.path.join(dir_output,'train') + dir_eval_flowing=os.path.join(dir_output,'eval') - - os.mkdir(dir_flow_train_imgs) - os.mkdir(dir_flow_train_labels) - - os.mkdir(dir_flow_eval_imgs) - os.mkdir(dir_flow_eval_labels) - - - - #set the gpu configuration - configuration() - - - #writing patches into a sub-folder in order to be flowed from directory. - provide_patches(dir_img,dir_seg,dir_flow_train_imgs, - dir_flow_train_labels, - input_height,input_width,blur_k,blur_aug, - flip_aug,binarization,scaling,scales,flip_index, - scaling_bluring,scaling_binarization,rotation, - augmentation=augmentation,patches=patches) - - provide_patches(dir_img_val,dir_seg_val,dir_flow_eval_imgs, - dir_flow_eval_labels, - input_height,input_width,blur_k,blur_aug, - flip_aug,binarization,scaling,scales,flip_index, - scaling_bluring,scaling_binarization,rotation, - augmentation=False,patches=patches) + dir_flow_train_imgs=os.path.join(dir_train_flowing,'images/') + dir_flow_train_labels=os.path.join(dir_train_flowing,'labels/') - if weighted_loss: - weights=np.zeros(n_classes) - for obj in os.listdir(dir_seg): - label_obj=cv2.imread(dir_seg+'/'+obj) - label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) - weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images/') + dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels/') + + if os.path.isdir(dir_train_flowing): + os.system('rm -rf '+dir_train_flowing) + os.makedirs(dir_train_flowing) + else: + os.makedirs(dir_train_flowing) + + if os.path.isdir(dir_eval_flowing): + os.system('rm -rf '+dir_eval_flowing) + os.makedirs(dir_eval_flowing) + else: + os.makedirs(dir_eval_flowing) - weights=1.00/weights + os.mkdir(dir_flow_train_imgs) + os.mkdir(dir_flow_train_labels) - weights=weights/float(np.sum(weights)) - weights=weights/float(np.min(weights)) - weights=weights/float(np.sum(weights)) + os.mkdir(dir_flow_eval_imgs) + os.mkdir(dir_flow_eval_labels) + + + #set the gpu configuration + configuration() - - + + #writing patches into a sub-folder in order to be flowed from directory. + provide_patches(dir_img,dir_seg,dir_flow_train_imgs, + dir_flow_train_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + rotation_not_90,thetha,scaling_flip, + augmentation=augmentation,patches=patches) - #get our model. - model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + provide_patches(dir_img_val,dir_seg_val,dir_flow_eval_imgs, + dir_flow_eval_labels, + input_height,input_width,blur_k,blur_aug, + flip_aug,binarization,scaling,scales,flip_index, + scaling_bluring,scaling_binarization,rotation, + rotation_not_90,thetha,scaling_flip, + augmentation=False,patches=patches) + + + continue_train = False + + if continue_train: + model_dir_start = '/home/vahid/Documents/struktur_full_data/output_multi/model_0.h5' + model = load_model (model_dir_start, compile = True, custom_objects={'soft_dice_loss': soft_dice_loss}) + index_start = 1 + else: + #get our model. + index_start = 0 + model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) #if you want to see the model structure just uncomment model summary. #model.summary() - if not weighted_loss: - model.compile(loss='categorical_crossentropy', - optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - if weighted_loss: - model.compile(loss=weighted_categorical_crossentropy(weights), - optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - - mc = keras.callbacks.ModelCheckpoint('weights{epoch:08d}.h5', - save_weights_only=True, period=1) - + + #model.compile(loss='categorical_crossentropy', + #optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + + model.compile(loss=soft_dice_loss, + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) #generating train and evaluation data train_gen = data_gen(dir_flow_train_imgs,dir_flow_train_labels, batch_size = n_batch, @@ -166,20 +171,20 @@ def run(n_classes,n_epochs,input_height, val_gen = data_gen(dir_flow_eval_imgs,dir_flow_eval_labels, batch_size = n_batch, input_height=input_height, input_width=input_width,n_classes=n_classes ) + for i in range(index_start, n_epochs+index_start): + model.fit_generator( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch)-1, + validation_data=val_gen, + validation_steps=1, + epochs=1) + model.save(dir_output+'/'+'model_'+str(i)+'.h5') - model.fit_generator( - train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch)-1, - validation_data=val_gen, - validation_steps=1, - epochs=n_epochs) - - os.system('rm -rf '+dir_train_flowing) os.system('rm -rf '+dir_eval_flowing) - model.save(dir_output+'/'+'model'+'.h5') + #model.save(dir_output+'/'+'model'+'.h5') diff --git a/train/utils.py b/train/utils.py index afdc9e5..a77444e 100644 --- a/train/utils.py +++ b/train/utils.py @@ -6,7 +6,8 @@ from scipy.ndimage.interpolation import map_coordinates from scipy.ndimage.filters import gaussian_filter import random from tqdm import tqdm - +import imutils +import math @@ -19,6 +20,79 @@ def bluring(img_in,kind): img_blur=cv2.blur(img_in,(5,5)) return img_blur +def elastic_transform(image, alpha, sigma,seedj, random_state=None): + + """Elastic deformation of images as described in [Simard2003]_. + .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for + Convolutional Neural Networks applied to Visual Document Analysis", in + Proc. of the International Conference on Document Analysis and + Recognition, 2003. + """ + if random_state is None: + random_state = np.random.RandomState(seedj) + + shape = image.shape + dx = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode="constant", cval=0) * alpha + dy = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode="constant", cval=0) * alpha + dz = np.zeros_like(dx) + + x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) + indices = np.reshape(y+dy, (-1, 1)), np.reshape(x+dx, (-1, 1)), np.reshape(z, (-1, 1)) + + distored_image = map_coordinates(image, indices, order=1, mode='reflect') + return distored_image.reshape(image.shape) + +def rotation_90(img): + img_rot=np.zeros((img.shape[1],img.shape[0],img.shape[2])) + img_rot[:,:,0]=img[:,:,0].T + img_rot[:,:,1]=img[:,:,1].T + img_rot[:,:,2]=img[:,:,2].T + return img_rot + +def rotatedRectWithMaxArea(w, h, angle): + """ + Given a rectangle of size wxh that has been rotated by 'angle' (in + radians), computes the width and height of the largest possible + axis-aligned rectangle (maximal area) within the rotated rectangle. + """ + if w <= 0 or h <= 0: + return 0,0 + + width_is_longer = w >= h + side_long, side_short = (w,h) if width_is_longer else (h,w) + + # since the solutions for angle, -angle and 180-angle are all the same, + # if suffices to look at the first quadrant and the absolute values of sin,cos: + sin_a, cos_a = abs(math.sin(angle)), abs(math.cos(angle)) + if side_short <= 2.*sin_a*cos_a*side_long or abs(sin_a-cos_a) < 1e-10: + # half constrained case: two crop corners touch the longer side, + # the other two corners are on the mid-line parallel to the longer line + x = 0.5*side_short + wr,hr = (x/sin_a,x/cos_a) if width_is_longer else (x/cos_a,x/sin_a) + else: + # fully constrained case: crop touches all 4 sides + cos_2a = cos_a*cos_a - sin_a*sin_a + wr,hr = (w*cos_a - h*sin_a)/cos_2a, (h*cos_a - w*sin_a)/cos_2a + + return wr,hr + +def rotate_max_area(image,rotated, rotated_label,angle): + """ image: cv2 image matrix object + angle: in degree + """ + wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], + math.radians(angle)) + h, w, _ = rotated.shape + y1 = h//2 - int(hr/2) + y2 = y1 + int(hr) + x1 = w//2 - int(wr/2) + x2 = x1 + int(wr) + return rotated[y1:y2, x1:x2],rotated_label[y1:y2, x1:x2] +def rotation_not_90_func(img,label,thetha): + rotated=imutils.rotate(img,thetha) + rotated_label=imutils.rotate(label,thetha) + return rotate_max_area(img, rotated,rotated_label,thetha) + def color_images(seg, n_classes): ann_u=range(n_classes) if len(np.shape(seg))==3: @@ -65,7 +139,7 @@ def IoU(Yi,y_predi): return mIoU def data_gen(img_folder, mask_folder, batch_size,input_height, input_width,n_classes): c = 0 - n = os.listdir(img_folder) #List of training images + n = [f for f in os.listdir(img_folder) if not f.startswith('.')]# os.listdir(img_folder) #List of training images random.shuffle(n) while True: img = np.zeros((batch_size, input_height, input_width, 3)).astype('float') @@ -73,18 +147,26 @@ def data_gen(img_folder, mask_folder, batch_size,input_height, input_width,n_cla for i in range(c, c+batch_size): #initially from 0 to 16, c = 0. #print(img_folder+'/'+n[i]) - filename=n[i].split('.')[0] - train_img = cv2.imread(img_folder+'/'+n[i])/255. - train_img = cv2.resize(train_img, (input_width, input_height),interpolation=cv2.INTER_NEAREST)# Read an image from folder and resize - - img[i-c] = train_img #add to array - img[0], img[1], and so on. - train_mask = cv2.imread(mask_folder+'/'+filename+'.png') - #print(mask_folder+'/'+filename+'.png') - #print(train_mask.shape) - train_mask = get_one_hot( resize_image(train_mask,input_height,input_width),input_height,input_width,n_classes) - #train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] - - mask[i-c] = train_mask + + try: + filename=n[i].split('.')[0] + + train_img = cv2.imread(img_folder+'/'+n[i])/255. + train_img = cv2.resize(train_img, (input_width, input_height),interpolation=cv2.INTER_NEAREST)# Read an image from folder and resize + + img[i-c] = train_img #add to array - img[0], img[1], and so on. + train_mask = cv2.imread(mask_folder+'/'+filename+'.png') + #print(mask_folder+'/'+filename+'.png') + #print(train_mask.shape) + train_mask = get_one_hot( resize_image(train_mask,input_height,input_width),input_height,input_width,n_classes) + #train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] + + mask[i-c] = train_mask + except: + img[i-c] = np.ones((input_height, input_width, 3)).astype('float') + mask[i-c] = np.zeros((input_height, input_width, n_classes)).astype('float') + + c+=batch_size if(c+batch_size>=len(os.listdir(img_folder))): @@ -104,16 +186,10 @@ def otsu_copy(img): img_r[:,:,1]=threshold1 img_r[:,:,2]=threshold1 return img_r - -def rotation_90(img): - img_rot=np.zeros((img.shape[1],img.shape[0],img.shape[2])) - img_rot[:,:,0]=img[:,:,0].T - img_rot[:,:,1]=img[:,:,1].T - img_rot[:,:,2]=img[:,:,2].T - return img_rot - def get_patches(dir_img_f,dir_seg_f,img,label,height,width,indexer): + if img.shape[0]int(nxf): + nxf=int(nxf)+1 + if nyf>int(nyf): + nyf=int(nyf)+1 + + nxf=int(nxf) + nyf=int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d=i*width_scale + index_x_u=(i+1)*width_scale + + index_y_d=j*height_scale + index_y_u=(j+1)*height_scale + + if index_x_u>img_w: + index_x_u=img_w + index_x_d=img_w-width_scale + if index_y_u>img_h: + index_y_u=img_h + index_y_d=img_h-height_scale + + + img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] + label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] + + #img_patch=resize_image(img_patch,height,width) + #label_patch=resize_image(label_patch,height,width) + + cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) + cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) + indexer+=1 + + return indexer def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, @@ -211,6 +366,7 @@ def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, input_height,input_width,blur_k,blur_aug, flip_aug,binarization,scaling,scales,flip_index, scaling_bluring,scaling_binarization,rotation, + rotation_not_90,thetha,scaling_flip, augmentation=False,patches=False): imgs_cv_train=np.array(os.listdir(dir_img)) @@ -218,25 +374,15 @@ def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, indexer=0 for im, seg_i in tqdm(zip(imgs_cv_train,segs_cv_train)): + #print(im, seg_i) img_name=im.split('.')[0] - + print(img_name,'img_name') if not patches: cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', resize_image(cv2.imread(dir_img+'/'+im),input_height,input_width ) ) cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width ) ) indexer+=1 if augmentation: - if rotation: - cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', - rotation_90( resize_image(cv2.imread(dir_img+'/'+im), - input_height,input_width) ) ) - - - cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png', - rotation_90 ( resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width) ) ) - indexer+=1 - if flip_aug: for f_i in flip_index: cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', @@ -270,10 +416,10 @@ def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, if patches: - + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer) + cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'), + input_height,input_width,indexer=indexer) if augmentation: @@ -284,29 +430,37 @@ def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, rotation_90( cv2.imread(dir_img+'/'+im) ), rotation_90( cv2.imread(dir_seg+'/'+img_name+'.png') ), input_height,input_width,indexer=indexer) + + if rotation_not_90: + + for thetha_i in thetha: + img_max_rotated,label_max_rotated=rotation_not_90_func(cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'),thetha_i) + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, + img_max_rotated, + label_max_rotated, + input_height,input_width,indexer=indexer) if flip_aug: for f_i in flip_index: - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, cv2.flip( cv2.imread(dir_img+'/'+im) , f_i), cv2.flip( cv2.imread(dir_seg+'/'+img_name+'.png') ,f_i), input_height,input_width,indexer=indexer) if blur_aug: for blur_i in blur_k: + indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, bluring( cv2.imread(dir_img+'/'+im) , blur_i), cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer) - + input_height,input_width,indexer=indexer) + if scaling: for sc_ind in scales: - indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, - cv2.imread(dir_img+'/'+im) , - cv2.imread(dir_seg+'/'+img_name+'.png'), + indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, + cv2.imread(dir_img+'/'+im) , + cv2.imread(dir_seg+'/'+img_name+'.png'), input_height,input_width,indexer=indexer,scaler=sc_ind) if binarization: - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, otsu_copy( cv2.imread(dir_img+'/'+im)), cv2.imread(dir_seg+'/'+img_name+'.png'), @@ -317,17 +471,26 @@ def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, if scaling_bluring: for sc_ind in scales: for blur_i in blur_k: - indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, + indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, bluring( cv2.imread(dir_img+'/'+im) , blur_i) , cv2.imread(dir_seg+'/'+img_name+'.png') , input_height,input_width,indexer=indexer,scaler=sc_ind) if scaling_binarization: for sc_ind in scales: - indexer=get_patches_num_scale(dir_flow_train_imgs,dir_flow_train_labels, - otsu_copy( cv2.imread(dir_img+'/'+im)) , - cv2.imread(dir_seg+'/'+img_name+'.png'), + indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, + otsu_copy( cv2.imread(dir_img+'/'+im)) , + cv2.imread(dir_seg+'/'+img_name+'.png'), input_height,input_width,indexer=indexer,scaler=sc_ind) + + if scaling_flip: + for sc_ind in scales: + for f_i in flip_index: + indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, + cv2.flip( cv2.imread(dir_img+'/'+im) , f_i) , + cv2.flip(cv2.imread(dir_seg+'/'+img_name+'.png') ,f_i) , + input_height,input_width,indexer=indexer,scaler=sc_ind) + From 8884b90f052c9d29d10dcce7f8636d41437181b8 Mon Sep 17 00:00:00 2001 From: vahid Date: Tue, 22 Jun 2021 18:47:59 -0400 Subject: [PATCH 021/374] continue training, losses and etc --- train/config_params.json | 14 +++++--- train/train.py | 77 ++++++++++++++++++++++++++++++---------- train/utils.py | 2 -- 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index d8f1ac5..eaa50e1 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,6 +1,6 @@ { "n_classes" : 3, - "n_epochs" : 1, + "n_epochs" : 2, "input_height" : 448, "input_width" : 672, "weight_decay" : 1e-6, @@ -8,16 +8,22 @@ "learning_rate": 1e-4, "patches" : true, "pretraining" : true, - "augmentation" : true, + "augmentation" : false, "flip_aug" : false, - "blur_aug" : true, - "scaling" : false, + "blur_aug" : false, + "scaling" : true, "binarization" : false, "scaling_bluring" : false, "scaling_binarization" : false, "scaling_flip" : false, "rotation": false, "rotation_not_90": false, + "continue_training": false, + "index_start": 0, + "dir_of_start_model": " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, "dir_train": "/home/vahid/Documents/handwrittens_train/train", "dir_eval": "/home/vahid/Documents/handwrittens_train/eval", "dir_output": "/home/vahid/Documents/handwrittens_train/output" diff --git a/train/train.py b/train/train.py index c256d83..0cc5ef3 100644 --- a/train/train.py +++ b/train/train.py @@ -9,6 +9,7 @@ from models import * from utils import * from metrics import * from keras.models import load_model +from tqdm import tqdm def configuration(): keras.backend.clear_session() @@ -61,19 +62,24 @@ def config_params(): blur_k=['blur','guass','median'] # Used in order to blur image. Used for augmentation. scales= [ 0.5, 2 ] # Scale patches with these scales. Used for augmentation. flip_index=[0,1,-1] # Flip image. Used for augmentation. - + continue_training = False # If + index_start = 0 + dir_of_start_model = '' + is_loss_soft_dice = False + weighted_loss = False + data_is_provided = False @ex.automain def run(n_classes,n_epochs,input_height, - input_width,weight_decay, + input_width,weight_decay,weighted_loss, + index_start,dir_of_start_model,is_loss_soft_dice, n_batch,patches,augmentation,flip_aug ,blur_aug,scaling, binarization, - blur_k,scales,dir_train, + blur_k,scales,dir_train,data_is_provided, scaling_bluring,scaling_binarization,rotation, - rotation_not_90,thetha,scaling_flip, + rotation_not_90,thetha,scaling_flip,continue_training, flip_index,dir_eval ,dir_output,pretraining,learning_rate): - data_is_provided = False if data_is_provided: dir_train_flowing=os.path.join(dir_output,'train') @@ -143,12 +149,43 @@ def run(n_classes,n_epochs,input_height, augmentation=False,patches=patches) - continue_train = False + + if weighted_loss: + weights=np.zeros(n_classes) + if data_is_provided: + for obj in os.listdir(dir_flow_train_labels): + try: + label_obj=cv2.imread(dir_flow_train_labels+'/'+obj) + label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) + weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + except: + pass + else: + + for obj in os.listdir(dir_seg): + try: + label_obj=cv2.imread(dir_seg+'/'+obj) + label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) + weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + except: + pass + - if continue_train: - model_dir_start = '/home/vahid/Documents/struktur_full_data/output_multi/model_0.h5' - model = load_model (model_dir_start, compile = True, custom_objects={'soft_dice_loss': soft_dice_loss}) - index_start = 1 + weights=1.00/weights + + weights=weights/float(np.sum(weights)) + weights=weights/float(np.min(weights)) + weights=weights/float(np.sum(weights)) + + + + if continue_training: + if is_loss_soft_dice: + model = load_model (dir_of_start_model, compile = True, custom_objects={'soft_dice_loss': soft_dice_loss}) + if weighted_loss: + model = load_model (dir_of_start_model, compile = True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + if not is_loss_soft_dice and not weighted_loss: + model = load_model (dir_of_start_model, compile = True) else: #get our model. index_start = 0 @@ -158,12 +195,16 @@ def run(n_classes,n_epochs,input_height, #model.summary() + if not is_loss_soft_dice and not weighted_loss: + model.compile(loss='categorical_crossentropy', + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + if is_loss_soft_dice: + model.compile(loss=soft_dice_loss, + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - #model.compile(loss='categorical_crossentropy', - #optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - - model.compile(loss=soft_dice_loss, - optimizer = Adam(lr=learning_rate),metrics=['accuracy']) + if weighted_loss: + model.compile(loss=weighted_categorical_crossentropy(weights), + optimizer = Adam(lr=learning_rate),metrics=['accuracy']) #generating train and evaluation data train_gen = data_gen(dir_flow_train_imgs,dir_flow_train_labels, batch_size = n_batch, @@ -171,7 +212,7 @@ def run(n_classes,n_epochs,input_height, val_gen = data_gen(dir_flow_eval_imgs,dir_flow_eval_labels, batch_size = n_batch, input_height=input_height, input_width=input_width,n_classes=n_classes ) - for i in range(index_start, n_epochs+index_start): + for i in tqdm(range(index_start, n_epochs+index_start)): model.fit_generator( train_gen, steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch)-1, @@ -181,8 +222,8 @@ def run(n_classes,n_epochs,input_height, model.save(dir_output+'/'+'model_'+str(i)+'.h5') - os.system('rm -rf '+dir_train_flowing) - os.system('rm -rf '+dir_eval_flowing) + #os.system('rm -rf '+dir_train_flowing) + #os.system('rm -rf '+dir_eval_flowing) #model.save(dir_output+'/'+'model'+'.h5') diff --git a/train/utils.py b/train/utils.py index a77444e..19ab46e 100644 --- a/train/utils.py +++ b/train/utils.py @@ -374,9 +374,7 @@ def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, indexer=0 for im, seg_i in tqdm(zip(imgs_cv_train,segs_cv_train)): - #print(im, seg_i) img_name=im.split('.')[0] - print(img_name,'img_name') if not patches: cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', resize_image(cv2.imread(dir_img+'/'+im),input_height,input_width ) ) cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width ) ) From 2d9ba854674db7169c3aceb4fca562b96bbed1f1 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 23 Jun 2021 07:25:49 -0400 Subject: [PATCH 022/374] Update README.md --- train/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/train/README.md b/train/README.md index d0d26d6..87a59ce 100644 --- a/train/README.md +++ b/train/README.md @@ -23,14 +23,16 @@ be ``0`` and ``1`` for each class and pixel. In the case of multiclass, just set ``n_classes`` to the number of classes you have and the try to produce the labels by pixels set from ``0 , 1 ,2 .., n_classes-1``. The labels format should be png. +Our lables are 3 channel png images but only information of first channel is used. +If you have an image label with height and width of 10, for a binary case the first channel should look like this: -If you have an image label for a binary case it should look like this: + Label: [ [1, 0, 0, 1, 1, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ..., + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] ] - Label: [ [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]], - [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] , - [[1 0 0 1], [1 0 0 1] ,[1 0 0 1]] ] - - This means that you have an image by `3*4*3` and `pixel[0,0]` belongs + This means that you have an image by `10*10*3` and `pixel[0,0]` belongs to class `1` and `pixel[0,1]` belongs to class `0`. ### Training , evaluation and output From 15407393e20a5c66556a0ab8e364f2206156ad27 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 23 Jun 2021 07:55:36 -0400 Subject: [PATCH 023/374] Update README.md --- train/README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/train/README.md b/train/README.md index 87a59ce..464a9a4 100644 --- a/train/README.md +++ b/train/README.md @@ -39,12 +39,15 @@ If you have an image label with height and width of 10, for a binary case the fi The train and evaluation folders should contain subfolders of images and labels. The output folder should be an empty folder where the output model will be written to. -### Patches -If you want to train your model with patches, the height and width of -the patches should be defined and also the number of batches (how many patches -should be seen by the model in each iteration). - -In the case that the model should see the image once, like page extraction, -patches should be set to ``false``. +### Parameter configuration +* patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be set to ``false``. +* n_batch: Number of batches at each iteration. +* n_classes: Number of classes. In the case of binary classification this should be 2. +* n_epochs: Number of epochs. +* input_height: This indicates the height of model's input. +* input_width: This indicates the width of model's input. +* weight_decay: Weight decay of l2 regularization of model layers. +* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. +* flip_aug: If ``true``, different types of filp will applied on image. Type of flips is given by "flip_index" in train.py file. From 491cdbf9342ffeebabe088b60371c2f18dd8cfaf Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 23 Jun 2021 08:21:12 -0400 Subject: [PATCH 024/374] Update README.md --- train/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 464a9a4..af8595f 100644 --- a/train/README.md +++ b/train/README.md @@ -48,6 +48,18 @@ The output folder should be an empty folder where the output model will be writt * input_width: This indicates the width of model's input. * weight_decay: Weight decay of l2 regularization of model layers. * augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. -* flip_aug: If ``true``, different types of filp will applied on image. Type of flips is given by "flip_index" in train.py file. +* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" in train.py file. +* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" in train.py file. +* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" in train.py file. +* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rothation angles are given with "thetha" in train.py file. +* rotation: If ``true``, 90 degree rotation will be applied on image. +* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. +* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. +* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. +* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. +* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set "index_start" to 3 to start naming model with index 3. +* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data should be in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resize and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. From 76c75d1365ee31e5637c763c89e664e7bbc45b0d Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 23 Jun 2021 08:22:03 -0400 Subject: [PATCH 025/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index af8595f..c38aea1 100644 --- a/train/README.md +++ b/train/README.md @@ -59,7 +59,7 @@ The output folder should be an empty folder where the output model will be writt * scaling_flip: If ``true``, combination of scaling and flip will be applied on image. * continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set "index_start" to 3 to start naming model with index 3. * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data should be in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resize and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. From 310a709ac7d2b1632580b53d6b4b3c127230e808 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 23 Jun 2021 08:23:20 -0400 Subject: [PATCH 026/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index c38aea1..5272def 100644 --- a/train/README.md +++ b/train/README.md @@ -60,6 +60,6 @@ The output folder should be an empty folder where the output model will be writt * continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set "index_start" to 3 to start naming model with index 3. * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` * data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". -* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resize and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. +* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. From b1c8bdf10624e3580c46105c2f323a0bc14b8178 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 29 Jun 2021 07:19:32 -0400 Subject: [PATCH 027/374] Update README.md --- train/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/train/README.md b/train/README.md index 5272def..363ba21 100644 --- a/train/README.md +++ b/train/README.md @@ -34,6 +34,8 @@ If you have an image label with height and width of 10, for a binary case the fi This means that you have an image by `10*10*3` and `pixel[0,0]` belongs to class `1` and `pixel[0,1]` belongs to class `0`. + + A small sample of training data for binarization experiment can be found here https://qurator-data.de/binarization_training_data_sample/ which contains images and lables folders. ### Training , evaluation and output The train and evaluation folders should contain subfolders of images and labels. From 49853bb291ff048874c8d0d8a4683968211b9ac8 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 29 Jun 2021 07:21:34 -0400 Subject: [PATCH 028/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 363ba21..529d7c7 100644 --- a/train/README.md +++ b/train/README.md @@ -35,7 +35,7 @@ If you have an image label with height and width of 10, for a binary case the fi This means that you have an image by `10*10*3` and `pixel[0,0]` belongs to class `1` and `pixel[0,1]` belongs to class `0`. - A small sample of training data for binarization experiment can be found here https://qurator-data.de/binarization_training_data_sample/ which contains images and lables folders. + A small sample of training data for binarization experiment can be found here [Training data sample](https://qurator-data.de/binarization_training_data_sample/) which contains images and lables folders. ### Training , evaluation and output The train and evaluation folders should contain subfolders of images and labels. From 09c0d5e318e1115b99dc3c9635179851370b54fe Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 29 Jun 2021 07:22:13 -0400 Subject: [PATCH 029/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 529d7c7..58f3eae 100644 --- a/train/README.md +++ b/train/README.md @@ -35,7 +35,7 @@ If you have an image label with height and width of 10, for a binary case the fi This means that you have an image by `10*10*3` and `pixel[0,0]` belongs to class `1` and `pixel[0,1]` belongs to class `0`. - A small sample of training data for binarization experiment can be found here [Training data sample](https://qurator-data.de/binarization_training_data_sample/) which contains images and lables folders. + A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/binarization_training_data_sample/) , which contains images and lables folders. ### Training , evaluation and output The train and evaluation folders should contain subfolders of images and labels. From bcc900be1732ac5c9a94d2d99e37673c745d96af Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 29 Jun 2021 07:22:34 -0400 Subject: [PATCH 030/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 58f3eae..0f0eb55 100644 --- a/train/README.md +++ b/train/README.md @@ -35,7 +35,7 @@ If you have an image label with height and width of 10, for a binary case the fi This means that you have an image by `10*10*3` and `pixel[0,0]` belongs to class `1` and `pixel[0,1]` belongs to class `0`. - A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/binarization_training_data_sample/) , which contains images and lables folders. + A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/binarization_training_data_sample/), which contains images and lables folders. ### Training , evaluation and output The train and evaluation folders should contain subfolders of images and labels. From 083f5ae881436fad4e3f0e5b2caac068fa7bcf54 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 14 Jul 2021 06:01:33 -0400 Subject: [PATCH 031/374] Update README.md --- train/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index 0f0eb55..8acfa12 100644 --- a/train/README.md +++ b/train/README.md @@ -35,7 +35,7 @@ If you have an image label with height and width of 10, for a binary case the fi This means that you have an image by `10*10*3` and `pixel[0,0]` belongs to class `1` and `pixel[0,1]` belongs to class `0`. - A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/binarization_training_data_sample/), which contains images and lables folders. + A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/~vahid.rezanezhad/binarization_training_data_sample/), which contains images and lables folders. ### Training , evaluation and output The train and evaluation folders should contain subfolders of images and labels. From 5282caa3286f121f9195263d5419c3876c7d9b4f Mon Sep 17 00:00:00 2001 From: vahid Date: Mon, 22 Aug 2022 13:03:10 +0200 Subject: [PATCH 032/374] supposed to solve https://github.com/qurator-spk/sbb_binarization/issues/41 --- ..._model_load_pretrained_weights_and_save.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 train/build_model_load_pretrained_weights_and_save.py diff --git a/train/build_model_load_pretrained_weights_and_save.py b/train/build_model_load_pretrained_weights_and_save.py new file mode 100644 index 0000000..251e698 --- /dev/null +++ b/train/build_model_load_pretrained_weights_and_save.py @@ -0,0 +1,33 @@ +import os +import sys +import tensorflow as tf +import keras , warnings +from keras.optimizers import * +from sacred import Experiment +from models import * +from utils import * +from metrics import * + + + + +def configuration(): + gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) + session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) + + +if __name__=='__main__': + n_classes = 2 + input_height = 224 + input_width = 448 + weight_decay = 1e-6 + pretraining = False + dir_of_weights = 'model_bin_sbb_ens.h5' + + #configuration() + + model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + model.load_weights(dir_of_weights) + model.save('./name_in_another_python_version.h5') + + From 57dae564b359f905f636bb4579aff12d7e336d36 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 4 Apr 2024 11:26:28 +0200 Subject: [PATCH 033/374] adjusting to tf2 --- ..._model_load_pretrained_weights_and_save.py | 4 ++-- train/metrics.py | 2 +- train/models.py | 8 +++---- train/train.py | 24 +++++++------------ 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/train/build_model_load_pretrained_weights_and_save.py b/train/build_model_load_pretrained_weights_and_save.py index 251e698..3b1a577 100644 --- a/train/build_model_load_pretrained_weights_and_save.py +++ b/train/build_model_load_pretrained_weights_and_save.py @@ -1,8 +1,8 @@ import os import sys import tensorflow as tf -import keras , warnings -from keras.optimizers import * +import warnings +from tensorflow.keras.optimizers import * from sacred import Experiment from models import * from utils import * diff --git a/train/metrics.py b/train/metrics.py index c63cc22..1768960 100644 --- a/train/metrics.py +++ b/train/metrics.py @@ -1,4 +1,4 @@ -from keras import backend as K +from tensorflow.keras import backend as K import tensorflow as tf import numpy as np diff --git a/train/models.py b/train/models.py index 7c806b4..40a21a1 100644 --- a/train/models.py +++ b/train/models.py @@ -1,7 +1,7 @@ -from keras.models import * -from keras.layers import * -from keras import layers -from keras.regularizers import l2 +from tensorflow.keras.models import * +from tensorflow.keras.layers import * +from tensorflow.keras import layers +from tensorflow.keras.regularizers import l2 resnet50_Weights_path='./pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' IMAGE_ORDERING ='channels_last' diff --git a/train/train.py b/train/train.py index 0cc5ef3..142b79b 100644 --- a/train/train.py +++ b/train/train.py @@ -1,29 +1,21 @@ import os import sys import tensorflow as tf -from keras.backend.tensorflow_backend import set_session -import keras , warnings -from keras.optimizers import * +from tensorflow.compat.v1.keras.backend import set_session +import warnings +from tensorflow.keras.optimizers import * from sacred import Experiment from models import * from utils import * from metrics import * -from keras.models import load_model +from tensorflow.keras.models import load_model from tqdm import tqdm def configuration(): - keras.backend.clear_session() - tf.reset_default_graph() - warnings.filterwarnings('ignore') - - os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' - config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) - - + config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True - config.gpu_options.per_process_gpu_memory_fraction=0.95#0.95 - config.gpu_options.visible_device_list="0" - set_session(tf.Session(config=config)) + session = tf.compat.v1.Session(config=config) + set_session(session) def get_dirs_or_files(input_data): if os.path.isdir(input_data): @@ -219,7 +211,7 @@ def run(n_classes,n_epochs,input_height, validation_data=val_gen, validation_steps=1, epochs=1) - model.save(dir_output+'/'+'model_'+str(i)+'.h5') + model.save(dir_output+'/'+'model_'+str(i)) #os.system('rm -rf '+dir_train_flowing) From ced1f851e267cf986d0e1dbf1bb63e15db31c823 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 4 Apr 2024 11:30:12 +0200 Subject: [PATCH 034/374] adding requirements --- train/requirements.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 train/requirements.txt diff --git a/train/requirements.txt b/train/requirements.txt new file mode 100644 index 0000000..f804172 --- /dev/null +++ b/train/requirements.txt @@ -0,0 +1,7 @@ +tensorflow == 2.12.1 +sacred +opencv-python +seaborn +tqdm +imutils + From 45652294972f2ce7c8d1f473621901f322b9c4b6 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:03:02 +0200 Subject: [PATCH 035/374] use headless cv2 --- train/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/train/requirements.txt b/train/requirements.txt index f804172..cbe2d88 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -1,7 +1,6 @@ tensorflow == 2.12.1 sacred -opencv-python +opencv-python-headless seaborn tqdm imutils - From d0b039505956af90594d14a6535add8deeaa8583 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 10 Apr 2024 20:26:26 +0200 Subject: [PATCH 036/374] add info on helpful tools (fix #14) --- train/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/train/README.md b/train/README.md index 8acfa12..89fa227 100644 --- a/train/README.md +++ b/train/README.md @@ -10,6 +10,16 @@ Either clone the repository via `git clone https://github.com/qurator-spk/sbb_pi ### Pretrained encoder Download our pretrained weights and add them to a ``pretrained_model`` folder: https://qurator-data.de/sbb_pixelwise_segmentation/pretrained_encoder/ + +### Helpful tools +* [`pagexml2img`](https://github.com/qurator-spk/page2img) +> Tool to extract 2-D or 3-D RGB images from PAGE-XML data. In the former case, the output will be 1 2-D image array which each class has filled with a pixel value. In the case of a 3-D RGB image, +each class will be defined with a RGB value and beside images, a text file of classes will also be produced. +* [`cocoSegmentationToPng`](https://github.com/nightrome/cocostuffapi/blob/17acf33aef3c6cc2d6aca46dcf084266c2778cf0/PythonAPI/pycocotools/cocostuffhelper.py#L130) +> Convert COCO GT or results for a single image to a segmentation map and write it to disk. +* [`ocrd-segment-extract-pages`](https://github.com/OCR-D/ocrd_segment/blob/master/ocrd_segment/extract_pages.py) +> Extract region classes and their colours in mask (pseg) images. Allows the color map as free dict parameter, and comes with a default that mimics PageViewer's coloring for quick debugging; it also warns when regions do overlap. + ## Usage ### Train From 39aa88669b98f364b33520ce45ff42b126be686c Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:40:23 +0200 Subject: [PATCH 037/374] update parameter config docs (fix #11) --- train/train.py | 57 +++++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/train/train.py b/train/train.py index 142b79b..9f833e0 100644 --- a/train/train.py +++ b/train/train.py @@ -29,37 +29,36 @@ ex = Experiment() @ex.config def config_params(): - n_classes=None # Number of classes. If your case study is binary case the set it to 2 and otherwise give your number of cases. - n_epochs=1 - input_height=224*1 - input_width=224*1 + n_classes=None # Number of classes. In the case of binary classification this should be 2. + n_epochs=1 # Number of epochs. + input_height=224*1 # Height of model's input in pixels. + input_width=224*1 # Width of model's input in pixels. weight_decay=1e-6 # Weight decay of l2 regularization of model layers. n_batch=1 # Number of batches at each iteration. - learning_rate=1e-4 - patches=False # Make patches of image in order to use all information of image. In the case of page - # extraction this should be set to false since model should see all image. - augmentation=False - flip_aug=False # Flip image (augmentation). - blur_aug=False # Blur patches of image (augmentation). - scaling=False # Scaling of patches (augmentation) will be imposed if this set to true. - binarization=False # Otsu thresholding. Used for augmentation in the case of binary case like textline prediction. For multicases should not be applied. - dir_train=None # Directory of training dataset (sub-folders should be named images and labels). - dir_eval=None # Directory of validation dataset (sub-folders should be named images and labels). - dir_output=None # Directory of output where the model should be saved. - pretraining=False # Set true to load pretrained weights of resnet50 encoder. - scaling_bluring=False - scaling_binarization=False - scaling_flip=False - thetha=[10,-10] - blur_k=['blur','guass','median'] # Used in order to blur image. Used for augmentation. - scales= [ 0.5, 2 ] # Scale patches with these scales. Used for augmentation. - flip_index=[0,1,-1] # Flip image. Used for augmentation. - continue_training = False # If - index_start = 0 - dir_of_start_model = '' - is_loss_soft_dice = False - weighted_loss = False - data_is_provided = False + learning_rate=1e-4 # Set the learning rate. + patches=False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. + augmentation=False # To apply any kind of augmentation, this parameter must be set to true. + flip_aug=False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in train.py. + blur_aug=False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in train.py. + scaling=False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in train.py. + binarization=False # If true, Otsu thresholding will be applied to augment the input with binarized images. + dir_train=None # Directory of training dataset with subdirectories having the names "images" and "labels". + dir_eval=None # Directory of validation dataset with subdirectories having the names "images" and "labels". + dir_output=None # Directory where the output model will be saved. + pretraining=False # Set to true to load pretrained weights of ResNet50 encoder. + scaling_bluring=False # If true, a combination of scaling and blurring will be applied to the image. + scaling_binarization=False # If true, a combination of scaling and binarization will be applied to the image. + scaling_flip=False # If true, a combination of scaling and flipping will be applied to the image. + thetha=[10,-10] # Rotate image by these angles for augmentation. + blur_k=['blur','gauss','median'] # Blur image for augmentation. + scales=[0.5,2] # Scale patches for augmentation. + flip_index=[0,1,-1] # Flip image for augmentation. + continue_training = False # Set to true if you would like to continue training an already trained a model. + index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. + dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. + is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. + weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. + data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". @ex.automain def run(n_classes,n_epochs,input_height, From 666a62622ee95f2c155eb6db6dfa58bd31f15971 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 10 Apr 2024 22:20:23 +0200 Subject: [PATCH 038/374] code formatting with black; typos --- train/README.md | 6 +- ..._model_load_pretrained_weights_and_save.py | 14 +- train/config_params.json | 6 +- train/metrics.py | 209 ++--- train/models.py | 237 +++--- train/requirements.txt | 2 + train/train.py | 272 +++---- train/utils.py | 763 +++++++++--------- 8 files changed, 741 insertions(+), 768 deletions(-) diff --git a/train/README.md b/train/README.md index 89fa227..899c9a3 100644 --- a/train/README.md +++ b/train/README.md @@ -48,7 +48,7 @@ If you have an image label with height and width of 10, for a binary case the fi A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/~vahid.rezanezhad/binarization_training_data_sample/), which contains images and lables folders. ### Training , evaluation and output -The train and evaluation folders should contain subfolders of images and labels. +The train and evaluation folders should contain subfolders of images and labels. The output folder should be an empty folder where the output model will be written to. ### Parameter configuration @@ -63,7 +63,7 @@ The output folder should be an empty folder where the output model will be writt * flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" in train.py file. * blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" in train.py file. * scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" in train.py file. -* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rothation angles are given with "thetha" in train.py file. +* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" in train.py file. * rotation: If ``true``, 90 degree rotation will be applied on image. * binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. * scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. @@ -73,5 +73,3 @@ The output folder should be an empty folder where the output model will be writt * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` * data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. - - diff --git a/train/build_model_load_pretrained_weights_and_save.py b/train/build_model_load_pretrained_weights_and_save.py index 3b1a577..125611e 100644 --- a/train/build_model_load_pretrained_weights_and_save.py +++ b/train/build_model_load_pretrained_weights_and_save.py @@ -9,25 +9,21 @@ from utils import * from metrics import * - - def configuration(): gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) -if __name__=='__main__': +if __name__ == '__main__': n_classes = 2 input_height = 224 input_width = 448 weight_decay = 1e-6 pretraining = False dir_of_weights = 'model_bin_sbb_ens.h5' - - #configuration() - - model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + + # configuration() + + model = resnet50_unet(n_classes, input_height, input_width, weight_decay, pretraining) model.load_weights(dir_of_weights) model.save('./name_in_another_python_version.h5') - - diff --git a/train/config_params.json b/train/config_params.json index eaa50e1..7505a81 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -24,7 +24,7 @@ "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "/home/vahid/Documents/handwrittens_train/train", - "dir_eval": "/home/vahid/Documents/handwrittens_train/eval", - "dir_output": "/home/vahid/Documents/handwrittens_train/output" + "dir_train": "/train", + "dir_eval": "/eval", + "dir_output": "/output" } diff --git a/train/metrics.py b/train/metrics.py index 1768960..cd30b02 100644 --- a/train/metrics.py +++ b/train/metrics.py @@ -2,8 +2,8 @@ from tensorflow.keras import backend as K import tensorflow as tf import numpy as np -def focal_loss(gamma=2., alpha=4.): +def focal_loss(gamma=2., alpha=4.): gamma = float(gamma) alpha = float(alpha) @@ -37,8 +37,10 @@ def focal_loss(gamma=2., alpha=4.): fl = tf.multiply(alpha, tf.multiply(weight, ce)) reduced_fl = tf.reduce_max(fl, axis=1) return tf.reduce_mean(reduced_fl) + return focal_loss_fixed + def weighted_categorical_crossentropy(weights=None): """ weighted_categorical_crossentropy @@ -50,117 +52,131 @@ def weighted_categorical_crossentropy(weights=None): def loss(y_true, y_pred): labels_floats = tf.cast(y_true, tf.float32) - per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) - + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats, logits=y_pred) + if weights is not None: weight_mask = tf.maximum(tf.reduce_max(tf.constant( np.array(weights, dtype=np.float32)[None, None, None]) - * labels_floats, axis=-1), 1.0) + * labels_floats, axis=-1), 1.0) per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] return tf.reduce_mean(per_pixel_loss) + return loss + + def image_categorical_cross_entropy(y_true, y_pred, weights=None): """ :param y_true: tensor of shape (batch_size, height, width) representing the ground truth. :param y_pred: tensor of shape (batch_size, height, width) representing the prediction. :return: The mean cross-entropy on softmaxed tensors. """ - + labels_floats = tf.cast(y_true, tf.float32) - per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) - + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats, logits=y_pred) + if weights is not None: weight_mask = tf.maximum( - tf.reduce_max(tf.constant( - np.array(weights, dtype=np.float32)[None, None, None]) - * labels_floats, axis=-1), 1.0) + tf.reduce_max(tf.constant( + np.array(weights, dtype=np.float32)[None, None, None]) + * labels_floats, axis=-1), 1.0) per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] - - return tf.reduce_mean(per_pixel_loss) -def class_tversky(y_true, y_pred): - smooth = 1.0#1.00 - y_true = K.permute_dimensions(y_true, (3,1,2,0)) - y_pred = K.permute_dimensions(y_pred, (3,1,2,0)) + return tf.reduce_mean(per_pixel_loss) + + +def class_tversky(y_true, y_pred): + smooth = 1.0 # 1.00 + + y_true = K.permute_dimensions(y_true, (3, 1, 2, 0)) + y_pred = K.permute_dimensions(y_pred, (3, 1, 2, 0)) y_true_pos = K.batch_flatten(y_true) y_pred_pos = K.batch_flatten(y_pred) true_pos = K.sum(y_true_pos * y_pred_pos, 1) - false_neg = K.sum(y_true_pos * (1-y_pred_pos), 1) - false_pos = K.sum((1-y_true_pos)*y_pred_pos, 1) - alpha = 0.2#0.5 - beta=0.8 - return (true_pos + smooth)/(true_pos + alpha*false_neg + (beta)*false_pos + smooth) + false_neg = K.sum(y_true_pos * (1 - y_pred_pos), 1) + false_pos = K.sum((1 - y_true_pos) * y_pred_pos, 1) + alpha = 0.2 # 0.5 + beta = 0.8 + return (true_pos + smooth) / (true_pos + alpha * false_neg + beta * false_pos + smooth) -def focal_tversky_loss(y_true,y_pred): + +def focal_tversky_loss(y_true, y_pred): pt_1 = class_tversky(y_true, y_pred) - gamma =1.3#4./3.0#1.3#4.0/3.00# 0.75 - return K.sum(K.pow((1-pt_1), gamma)) + gamma = 1.3 # 4./3.0#1.3#4.0/3.00# 0.75 + return K.sum(K.pow((1 - pt_1), gamma)) + def generalized_dice_coeff2(y_true, y_pred): n_el = 1 - for dim in y_true.shape: + for dim in y_true.shape: n_el *= int(dim) n_cl = y_true.shape[-1] w = K.zeros(shape=(n_cl,)) - w = (K.sum(y_true, axis=(0,1,2)))/(n_el) - w = 1/(w**2+0.000001) - numerator = y_true*y_pred - numerator = w*K.sum(numerator,(0,1,2)) + w = (K.sum(y_true, axis=(0, 1, 2))) / n_el + w = 1 / (w ** 2 + 0.000001) + numerator = y_true * y_pred + numerator = w * K.sum(numerator, (0, 1, 2)) numerator = K.sum(numerator) - denominator = y_true+y_pred - denominator = w*K.sum(denominator,(0,1,2)) + denominator = y_true + y_pred + denominator = w * K.sum(denominator, (0, 1, 2)) denominator = K.sum(denominator) - return 2*numerator/denominator + return 2 * numerator / denominator + + def generalized_dice_coeff(y_true, y_pred): - axes = tuple(range(1, len(y_pred.shape)-1)) + axes = tuple(range(1, len(y_pred.shape) - 1)) Ncl = y_pred.shape[-1] w = K.zeros(shape=(Ncl,)) w = K.sum(y_true, axis=axes) - w = 1/(w**2+0.000001) + w = 1 / (w ** 2 + 0.000001) # Compute gen dice coef: - numerator = y_true*y_pred - numerator = w*K.sum(numerator,axes) + numerator = y_true * y_pred + numerator = w * K.sum(numerator, axes) numerator = K.sum(numerator) - denominator = y_true+y_pred - denominator = w*K.sum(denominator,axes) + denominator = y_true + y_pred + denominator = w * K.sum(denominator, axes) denominator = K.sum(denominator) - gen_dice_coef = 2*numerator/denominator + gen_dice_coef = 2 * numerator / denominator return gen_dice_coef + def generalized_dice_loss(y_true, y_pred): return 1 - generalized_dice_coeff2(y_true, y_pred) -def soft_dice_loss(y_true, y_pred, epsilon=1e-6): - ''' + + +def soft_dice_loss(y_true, y_pred, epsilon=1e-6): + """ Soft dice loss calculation for arbitrary batch size, number of classes, and number of spatial dimensions. Assumes the `channels_last` format. - + # Arguments y_true: b x X x Y( x Z...) x c One hot encoding of ground truth - y_pred: b x X x Y( x Z...) x c Network output, must sum to 1 over c channel (such as after softmax) + y_pred: b x X x Y( x Z...) x c Network output, must sum to 1 over c channel (such as after softmax) epsilon: Used for numerical stability to avoid divide by zero errors - + # References - V-Net: Fully Convolutional Neural Networks for Volumetric Medical Image Segmentation + V-Net: Fully Convolutional Neural Networks for Volumetric Medical Image Segmentation https://arxiv.org/abs/1606.04797 - More details on Dice loss formulation + More details on Dice loss formulation https://mediatum.ub.tum.de/doc/1395260/1395260.pdf (page 72) - + Adapted from https://github.com/Lasagne/Recipes/issues/99#issuecomment-347775022 - ''' - + """ + # skip the batch and class axis for calculating Dice score - axes = tuple(range(1, len(y_pred.shape)-1)) - + axes = tuple(range(1, len(y_pred.shape) - 1)) + numerator = 2. * K.sum(y_pred * y_true, axes) denominator = K.sum(K.square(y_pred) + K.square(y_true), axes) - return 1.00 - K.mean(numerator / (denominator + epsilon)) # average over classes and batch + return 1.00 - K.mean(numerator / (denominator + epsilon)) # average over classes and batch -def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last = True, mean_per_class=False, verbose=False): + +def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last=True, mean_per_class=False, + verbose=False): """ Compute mean metrics of two segmentation masks, via Keras. @@ -193,13 +209,13 @@ def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last = H = height, N = number of classes """ - + flag_soft = (metric_type == 'soft') flag_naive_mean = (metric_type == 'naive') - + # always assume one or more classes num_classes = K.shape(y_true)[-1] - + if not flag_soft: # get one-hot encoded masks from y_pred (true masks should already be one-hot) y_pred = K.one_hot(K.argmax(y_pred), num_classes) @@ -211,29 +227,29 @@ def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last = y_pred = K.cast(y_pred, 'float32') # intersection and union shapes are batch_size * n_classes (values = area in pixels) - axes = (1,2) # W,H axes of each image + axes = (1, 2) # W,H axes of each image intersection = K.sum(K.abs(y_true * y_pred), axis=axes) mask_sum = K.sum(K.abs(y_true), axis=axes) + K.sum(K.abs(y_pred), axis=axes) - union = mask_sum - intersection # or, np.logical_or(y_pred, y_true) for one-hot + union = mask_sum - intersection # or, np.logical_or(y_pred, y_true) for one-hot smooth = .001 iou = (intersection + smooth) / (union + smooth) - dice = 2 * (intersection + smooth)/(mask_sum + smooth) + dice = 2 * (intersection + smooth) / (mask_sum + smooth) metric = {'iou': iou, 'dice': dice}[metric_name] # define mask to be 0 when no pixels are present in either y_true or y_pred, 1 otherwise - mask = K.cast(K.not_equal(union, 0), 'float32') - + mask = K.cast(K.not_equal(union, 0), 'float32') + if drop_last: - metric = metric[:,:-1] - mask = mask[:,:-1] - + metric = metric[:, :-1] + mask = mask[:, :-1] + if verbose: print('intersection, union') print(K.eval(intersection), K.eval(union)) - print(K.eval(intersection/union)) - + print(K.eval(intersection / union)) + # return mean metrics: remaining axes are (batch, classes) if flag_naive_mean: return K.mean(metric) @@ -243,13 +259,14 @@ def seg_metrics(y_true, y_pred, metric_name, metric_type='standard', drop_last = non_zero = tf.greater(class_count, 0) non_zero_sum = tf.boolean_mask(K.sum(metric * mask, axis=0), non_zero) non_zero_count = tf.boolean_mask(class_count, non_zero) - + if verbose: print('Counts of inputs with class present, metrics for non-absent classes') print(K.eval(class_count), K.eval(non_zero_sum / non_zero_count)) - + return K.mean(non_zero_sum / non_zero_count) + def mean_iou(y_true, y_pred, **kwargs): """ Compute mean Intersection over Union of two segmentation masks, via Keras. @@ -257,65 +274,69 @@ def mean_iou(y_true, y_pred, **kwargs): Calls metrics_k(y_true, y_pred, metric_name='iou'), see there for allowed kwargs. """ return seg_metrics(y_true, y_pred, metric_name='iou', **kwargs) + + def Mean_IOU(y_true, y_pred): nb_classes = K.int_shape(y_pred)[-1] iou = [] true_pixels = K.argmax(y_true, axis=-1) pred_pixels = K.argmax(y_pred, axis=-1) void_labels = K.equal(K.sum(y_true, axis=-1), 0) - for i in range(0, nb_classes): # exclude first label (background) and last label (void) - true_labels = K.equal(true_pixels, i)# & ~void_labels - pred_labels = K.equal(pred_pixels, i)# & ~void_labels + for i in range(0, nb_classes): # exclude first label (background) and last label (void) + true_labels = K.equal(true_pixels, i) # & ~void_labels + pred_labels = K.equal(pred_pixels, i) # & ~void_labels inter = tf.to_int32(true_labels & pred_labels) union = tf.to_int32(true_labels | pred_labels) - legal_batches = K.sum(tf.to_int32(true_labels), axis=1)>0 - ious = K.sum(inter, axis=1)/K.sum(union, axis=1) - iou.append(K.mean(tf.gather(ious, indices=tf.where(legal_batches)))) # returns average IoU of the same objects + legal_batches = K.sum(tf.to_int32(true_labels), axis=1) > 0 + ious = K.sum(inter, axis=1) / K.sum(union, axis=1) + iou.append(K.mean(tf.gather(ious, indices=tf.where(legal_batches)))) # returns average IoU of the same objects iou = tf.stack(iou) legal_labels = ~tf.debugging.is_nan(iou) iou = tf.gather(iou, indices=tf.where(legal_labels)) return K.mean(iou) + def iou_vahid(y_true, y_pred): - nb_classes = tf.shape(y_true)[-1]+tf.to_int32(1) + nb_classes = tf.shape(y_true)[-1] + tf.to_int32(1) true_pixels = K.argmax(y_true, axis=-1) pred_pixels = K.argmax(y_pred, axis=-1) iou = [] - + for i in tf.range(nb_classes): - tp=K.sum( tf.to_int32( K.equal(true_pixels, i) & K.equal(pred_pixels, i) ) ) - fp=K.sum( tf.to_int32( K.not_equal(true_pixels, i) & K.equal(pred_pixels, i) ) ) - fn=K.sum( tf.to_int32( K.equal(true_pixels, i) & K.not_equal(pred_pixels, i) ) ) - iouh=tp/(tp+fp+fn) + tp = K.sum(tf.to_int32(K.equal(true_pixels, i) & K.equal(pred_pixels, i))) + fp = K.sum(tf.to_int32(K.not_equal(true_pixels, i) & K.equal(pred_pixels, i))) + fn = K.sum(tf.to_int32(K.equal(true_pixels, i) & K.not_equal(pred_pixels, i))) + iouh = tp / (tp + fp + fn) iou.append(iouh) return K.mean(iou) - - -def IoU_metric(Yi,y_predi): - ## mean Intersection over Union - ## Mean IoU = TP/(FN + TP + FP) + + +def IoU_metric(Yi, y_predi): + # mean Intersection over Union + # Mean IoU = TP/(FN + TP + FP) y_predi = np.argmax(y_predi, axis=3) y_testi = np.argmax(Yi, axis=3) IoUs = [] Nclass = int(np.max(Yi)) + 1 for c in range(Nclass): - TP = np.sum( (Yi == c)&(y_predi==c) ) - FP = np.sum( (Yi != c)&(y_predi==c) ) - FN = np.sum( (Yi == c)&(y_predi != c)) - IoU = TP/float(TP + FP + FN) + TP = np.sum((Yi == c) & (y_predi == c)) + FP = np.sum((Yi != c) & (y_predi == c)) + FN = np.sum((Yi == c) & (y_predi != c)) + IoU = TP / float(TP + FP + FN) IoUs.append(IoU) - return K.cast( np.mean(IoUs) ,dtype='float32' ) + return K.cast(np.mean(IoUs), dtype='float32') def IoU_metric_keras(y_true, y_pred): - ## mean Intersection over Union - ## Mean IoU = TP/(FN + TP + FP) + # mean Intersection over Union + # Mean IoU = TP/(FN + TP + FP) init = tf.global_variables_initializer() sess = tf.Session() sess.run(init) - + return IoU_metric(y_true.eval(session=sess), y_pred.eval(session=sess)) + def jaccard_distance_loss(y_true, y_pred, smooth=100): """ Jaccard = (|X & Y|)/ (|X|+ |Y| - |X & Y|) @@ -334,5 +355,3 @@ def jaccard_distance_loss(y_true, y_pred, smooth=100): sum_ = K.sum(K.abs(y_true) + K.abs(y_pred), axis=-1) jac = (intersection + smooth) / (sum_ - intersection + smooth) return (1 - jac) * smooth - - diff --git a/train/models.py b/train/models.py index 40a21a1..f06823e 100644 --- a/train/models.py +++ b/train/models.py @@ -3,19 +3,20 @@ from tensorflow.keras.layers import * from tensorflow.keras import layers from tensorflow.keras.regularizers import l2 -resnet50_Weights_path='./pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' -IMAGE_ORDERING ='channels_last' -MERGE_AXIS=-1 +resnet50_Weights_path = './pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' +IMAGE_ORDERING = 'channels_last' +MERGE_AXIS = -1 -def one_side_pad( x ): +def one_side_pad(x): x = ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING)(x) if IMAGE_ORDERING == 'channels_first': - x = Lambda(lambda x : x[: , : , :-1 , :-1 ] )(x) + x = Lambda(lambda x: x[:, :, :-1, :-1])(x) elif IMAGE_ORDERING == 'channels_last': - x = Lambda(lambda x : x[: , :-1 , :-1 , : ] )(x) + x = Lambda(lambda x: x[:, :-1, :-1, :])(x) return x + def identity_block(input_tensor, kernel_size, filters, stage, block): """The identity block is the block that has no conv layer at shortcut. # Arguments @@ -28,7 +29,7 @@ def identity_block(input_tensor, kernel_size, filters, stage, block): Output tensor for the block. """ filters1, filters2, filters3 = filters - + if IMAGE_ORDERING == 'channels_last': bn_axis = 3 else: @@ -37,16 +38,16 @@ def identity_block(input_tensor, kernel_size, filters, stage, block): conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' - x = Conv2D(filters1, (1, 1) , data_format=IMAGE_ORDERING , name=conv_name_base + '2a')(input_tensor) + x = Conv2D(filters1, (1, 1), data_format=IMAGE_ORDERING, name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) - x = Conv2D(filters2, kernel_size , data_format=IMAGE_ORDERING , + x = Conv2D(filters2, kernel_size, data_format=IMAGE_ORDERING, padding='same', name=conv_name_base + '2b')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = Activation('relu')(x) - x = Conv2D(filters3 , (1, 1), data_format=IMAGE_ORDERING , name=conv_name_base + '2c')(x) + x = Conv2D(filters3, (1, 1), data_format=IMAGE_ORDERING, name=conv_name_base + '2c')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) x = layers.add([x, input_tensor]) @@ -68,7 +69,7 @@ def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)) And the shortcut should have strides=(2,2) as well """ filters1, filters2, filters3 = filters - + if IMAGE_ORDERING == 'channels_last': bn_axis = 3 else: @@ -77,20 +78,20 @@ def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)) conv_name_base = 'res' + str(stage) + block + '_branch' bn_name_base = 'bn' + str(stage) + block + '_branch' - x = Conv2D(filters1, (1, 1) , data_format=IMAGE_ORDERING , strides=strides, + x = Conv2D(filters1, (1, 1), data_format=IMAGE_ORDERING, strides=strides, name=conv_name_base + '2a')(input_tensor) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2a')(x) x = Activation('relu')(x) - x = Conv2D(filters2, kernel_size , data_format=IMAGE_ORDERING , padding='same', + x = Conv2D(filters2, kernel_size, data_format=IMAGE_ORDERING, padding='same', name=conv_name_base + '2b')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2b')(x) x = Activation('relu')(x) - x = Conv2D(filters3, (1, 1) , data_format=IMAGE_ORDERING , name=conv_name_base + '2c')(x) + x = Conv2D(filters3, (1, 1), data_format=IMAGE_ORDERING, name=conv_name_base + '2c')(x) x = BatchNormalization(axis=bn_axis, name=bn_name_base + '2c')(x) - shortcut = Conv2D(filters3, (1, 1) , data_format=IMAGE_ORDERING , strides=strides, + shortcut = Conv2D(filters3, (1, 1), data_format=IMAGE_ORDERING, strides=strides, name=conv_name_base + '1')(input_tensor) shortcut = BatchNormalization(axis=bn_axis, name=bn_name_base + '1')(shortcut) @@ -99,12 +100,11 @@ def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)) return x -def resnet50_unet_light(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): - assert input_height%32 == 0 - assert input_width%32 == 0 +def resnet50_unet_light(n_classes, input_height=224, input_width=224, weight_decay=1e-6, pretraining=False): + assert input_height % 32 == 0 + assert input_width % 32 == 0 - - img_input = Input(shape=(input_height,input_width , 3 )) + img_input = Input(shape=(input_height, input_width, 3)) if IMAGE_ORDERING == 'channels_last': bn_axis = 3 @@ -112,25 +112,24 @@ def resnet50_unet_light(n_classes,input_height=224,input_width=224,weight_decay= bn_axis = 1 x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) - x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2), kernel_regularizer=l2(weight_decay), + name='conv1')(x) f1 = x x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = Activation('relu')(x) - x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) - + x = MaxPooling2D((3, 3), data_format=IMAGE_ORDERING, strides=(2, 2))(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') - f2 = one_side_pad(x ) - + f2 = one_side_pad(x) x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') - f3 = x + f3 = x x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') @@ -138,85 +137,72 @@ def resnet50_unet_light(n_classes,input_height=224,input_width=224,weight_decay= x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') - f4 = x + f4 = x x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') - f5 = x - + f5 = x if pretraining: - model=Model( img_input , x ).load_weights(resnet50_Weights_path) + model = Model(img_input, x).load_weights(resnet50_Weights_path) - - v512_2048 = Conv2D( 512 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f5 ) - v512_2048 = ( BatchNormalization(axis=bn_axis))(v512_2048) + v512_2048 = Conv2D(512, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(f5) + v512_2048 = (BatchNormalization(axis=bn_axis))(v512_2048) v512_2048 = Activation('relu')(v512_2048) - - - v512_1024=Conv2D( 512 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f4 ) - v512_1024 = ( BatchNormalization(axis=bn_axis))(v512_1024) + v512_1024 = Conv2D(512, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(f4) + v512_1024 = (BatchNormalization(axis=bn_axis))(v512_1024) v512_1024 = Activation('relu')(v512_1024) - - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(v512_2048) - o = ( concatenate([ o ,v512_1024],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) - o = ( Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(v512_2048) + o = (concatenate([o, v512_1024], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([ o ,f3],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) - o = ( Conv2D( 256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f3], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(256, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([o,f2],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING))(o) - o = ( Conv2D( 128 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay) ) )(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f2], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(128, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([o,f1],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) - o = ( Conv2D( 64 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f1], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(64, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([o,img_input],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) - o = ( Conv2D( 32 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, img_input], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(32, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - - o = Conv2D( n_classes , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( o ) - o = ( BatchNormalization(axis=bn_axis))(o) + o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = (Activation('softmax'))(o) - - model = Model( img_input , o ) + model = Model(img_input, o) return model -def resnet50_unet(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): - assert input_height%32 == 0 - assert input_width%32 == 0 - - img_input = Input(shape=(input_height,input_width , 3 )) +def resnet50_unet(n_classes, input_height=224, input_width=224, weight_decay=1e-6, pretraining=False): + assert input_height % 32 == 0 + assert input_width % 32 == 0 + + img_input = Input(shape=(input_height, input_width, 3)) if IMAGE_ORDERING == 'channels_last': bn_axis = 3 @@ -224,25 +210,24 @@ def resnet50_unet(n_classes,input_height=224,input_width=224,weight_decay=1e-6,p bn_axis = 1 x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) - x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2), kernel_regularizer=l2(weight_decay), + name='conv1')(x) f1 = x x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = Activation('relu')(x) - x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) - + x = MaxPooling2D((3, 3), data_format=IMAGE_ORDERING, strides=(2, 2))(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') - f2 = one_side_pad(x ) - + f2 = one_side_pad(x) x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') - f3 = x + f3 = x x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') @@ -250,68 +235,60 @@ def resnet50_unet(n_classes,input_height=224,input_width=224,weight_decay=1e-6,p x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') - f4 = x + f4 = x x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') - f5 = x + f5 = x if pretraining: - Model( img_input , x ).load_weights(resnet50_Weights_path) + Model(img_input, x).load_weights(resnet50_Weights_path) - v1024_2048 = Conv2D( 1024 , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( f5 ) - v1024_2048 = ( BatchNormalization(axis=bn_axis))(v1024_2048) + v1024_2048 = Conv2D(1024, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))( + f5) + v1024_2048 = (BatchNormalization(axis=bn_axis))(v1024_2048) v1024_2048 = Activation('relu')(v1024_2048) - - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(v1024_2048) - o = ( concatenate([ o ,f4],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) - o = ( Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(v1024_2048) + o = (concatenate([o, f4], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([ o ,f3],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D( (1,1), data_format=IMAGE_ORDERING))(o) - o = ( Conv2D( 256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f3], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(256, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([o,f2],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING))(o) - o = ( Conv2D( 128 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay) ) )(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f2], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(128, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([o,f1],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) - o = ( Conv2D( 64 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f1], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(64, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - o = ( UpSampling2D( (2,2), data_format=IMAGE_ORDERING))(o) - o = ( concatenate([o,img_input],axis=MERGE_AXIS ) ) - o = ( ZeroPadding2D((1,1) , data_format=IMAGE_ORDERING ))(o) - o = ( Conv2D( 32 , (3, 3), padding='valid' , data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) ))(o) - o = ( BatchNormalization(axis=bn_axis))(o) + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, img_input], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(32, (3, 3), padding='valid', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = Activation('relu')(o) - - - o = Conv2D( n_classes , (1, 1) , padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay) )( o ) - o = ( BatchNormalization(axis=bn_axis))(o) + + o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(o) + o = (BatchNormalization(axis=bn_axis))(o) o = (Activation('softmax'))(o) - - model = Model( img_input , o ) - - + model = Model(img_input, o) return model diff --git a/train/requirements.txt b/train/requirements.txt index cbe2d88..20b6a32 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -4,3 +4,5 @@ opencv-python-headless seaborn tqdm imutils +numpy +scipy diff --git a/train/train.py b/train/train.py index 9f833e0..03faf46 100644 --- a/train/train.py +++ b/train/train.py @@ -11,12 +11,14 @@ from metrics import * from tensorflow.keras.models import load_model from tqdm import tqdm + def configuration(): config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True session = tf.compat.v1.Session(config=config) set_session(session) + def get_dirs_or_files(input_data): if os.path.isdir(input_data): image_input, labels_input = os.path.join(input_data, 'images/'), os.path.join(input_data, 'labels/') @@ -25,205 +27,187 @@ def get_dirs_or_files(input_data): assert os.path.isdir(labels_input), "{} is not a directory".format(labels_input) return image_input, labels_input + ex = Experiment() + @ex.config def config_params(): - n_classes=None # Number of classes. In the case of binary classification this should be 2. - n_epochs=1 # Number of epochs. - input_height=224*1 # Height of model's input in pixels. - input_width=224*1 # Width of model's input in pixels. - weight_decay=1e-6 # Weight decay of l2 regularization of model layers. - n_batch=1 # Number of batches at each iteration. - learning_rate=1e-4 # Set the learning rate. - patches=False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. - augmentation=False # To apply any kind of augmentation, this parameter must be set to true. - flip_aug=False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in train.py. - blur_aug=False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in train.py. - scaling=False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in train.py. - binarization=False # If true, Otsu thresholding will be applied to augment the input with binarized images. - dir_train=None # Directory of training dataset with subdirectories having the names "images" and "labels". - dir_eval=None # Directory of validation dataset with subdirectories having the names "images" and "labels". - dir_output=None # Directory where the output model will be saved. - pretraining=False # Set to true to load pretrained weights of ResNet50 encoder. - scaling_bluring=False # If true, a combination of scaling and blurring will be applied to the image. - scaling_binarization=False # If true, a combination of scaling and binarization will be applied to the image. - scaling_flip=False # If true, a combination of scaling and flipping will be applied to the image. - thetha=[10,-10] # Rotate image by these angles for augmentation. - blur_k=['blur','gauss','median'] # Blur image for augmentation. - scales=[0.5,2] # Scale patches for augmentation. - flip_index=[0,1,-1] # Flip image for augmentation. - continue_training = False # Set to true if you would like to continue training an already trained a model. - index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. - dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. - is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. - weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. - data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". + n_classes = None # Number of classes. In the case of binary classification this should be 2. + n_epochs = 1 # Number of epochs. + input_height = 224 * 1 # Height of model's input in pixels. + input_width = 224 * 1 # Width of model's input in pixels. + weight_decay = 1e-6 # Weight decay of l2 regularization of model layers. + n_batch = 1 # Number of batches at each iteration. + learning_rate = 1e-4 # Set the learning rate. + patches = False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. + augmentation = False # To apply any kind of augmentation, this parameter must be set to true. + flip_aug = False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in train.py. + blur_aug = False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in train.py. + scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in train.py. + binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. + dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". + dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". + dir_output = None # Directory where the output model will be saved. + pretraining = False # Set to true to load pretrained weights of ResNet50 encoder. + scaling_bluring = False # If true, a combination of scaling and blurring will be applied to the image. + scaling_binarization = False # If true, a combination of scaling and binarization will be applied to the image. + scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image. + thetha = [10, -10] # Rotate image by these angles for augmentation. + blur_k = ['blur', 'gauss', 'median'] # Blur image for augmentation. + scales = [0.5, 2] # Scale patches for augmentation. + flip_index = [0, 1, -1] # Flip image for augmentation. + continue_training = False # Set to true if you would like to continue training an already trained a model. + index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. + dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. + is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. + weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. + data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". + @ex.automain -def run(n_classes,n_epochs,input_height, - input_width,weight_decay,weighted_loss, - index_start,dir_of_start_model,is_loss_soft_dice, - n_batch,patches,augmentation,flip_aug - ,blur_aug,scaling, binarization, - blur_k,scales,dir_train,data_is_provided, - scaling_bluring,scaling_binarization,rotation, - rotation_not_90,thetha,scaling_flip,continue_training, - flip_index,dir_eval ,dir_output,pretraining,learning_rate): - - +def run(n_classes, n_epochs, input_height, + input_width, weight_decay, weighted_loss, + index_start, dir_of_start_model, is_loss_soft_dice, + n_batch, patches, augmentation, flip_aug, + blur_aug, scaling, binarization, + blur_k, scales, dir_train, data_is_provided, + scaling_bluring, scaling_binarization, rotation, + rotation_not_90, thetha, scaling_flip, continue_training, + flip_index, dir_eval, dir_output, pretraining, learning_rate): if data_is_provided: - dir_train_flowing=os.path.join(dir_output,'train') - dir_eval_flowing=os.path.join(dir_output,'eval') - - dir_flow_train_imgs=os.path.join(dir_train_flowing,'images') - dir_flow_train_labels=os.path.join(dir_train_flowing,'labels') - - dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images') - dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels') - + dir_train_flowing = os.path.join(dir_output, 'train') + dir_eval_flowing = os.path.join(dir_output, 'eval') + + dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') + dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') + + dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images') + dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels') + configuration() - + else: - dir_img,dir_seg=get_dirs_or_files(dir_train) - dir_img_val,dir_seg_val=get_dirs_or_files(dir_eval) - + dir_img, dir_seg = get_dirs_or_files(dir_train) + dir_img_val, dir_seg_val = get_dirs_or_files(dir_eval) + # make first a directory in output for both training and evaluations in order to flow data from these directories. - dir_train_flowing=os.path.join(dir_output,'train') - dir_eval_flowing=os.path.join(dir_output,'eval') - - dir_flow_train_imgs=os.path.join(dir_train_flowing,'images/') - dir_flow_train_labels=os.path.join(dir_train_flowing,'labels/') - - dir_flow_eval_imgs=os.path.join(dir_eval_flowing,'images/') - dir_flow_eval_labels=os.path.join(dir_eval_flowing,'labels/') - + dir_train_flowing = os.path.join(dir_output, 'train') + dir_eval_flowing = os.path.join(dir_output, 'eval') + + dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images/') + dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels/') + + dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images/') + dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels/') + if os.path.isdir(dir_train_flowing): - os.system('rm -rf '+dir_train_flowing) + os.system('rm -rf ' + dir_train_flowing) os.makedirs(dir_train_flowing) else: os.makedirs(dir_train_flowing) - + if os.path.isdir(dir_eval_flowing): - os.system('rm -rf '+dir_eval_flowing) + os.system('rm -rf ' + dir_eval_flowing) os.makedirs(dir_eval_flowing) else: os.makedirs(dir_eval_flowing) - os.mkdir(dir_flow_train_imgs) os.mkdir(dir_flow_train_labels) - + os.mkdir(dir_flow_eval_imgs) os.mkdir(dir_flow_eval_labels) - - - #set the gpu configuration + + # set the gpu configuration configuration() - - #writing patches into a sub-folder in order to be flowed from directory. - provide_patches(dir_img,dir_seg,dir_flow_train_imgs, + # writing patches into a sub-folder in order to be flowed from directory. + provide_patches(dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, - input_height,input_width,blur_k,blur_aug, - flip_aug,binarization,scaling,scales,flip_index, - scaling_bluring,scaling_binarization,rotation, - rotation_not_90,thetha,scaling_flip, - augmentation=augmentation,patches=patches) - - provide_patches(dir_img_val,dir_seg_val,dir_flow_eval_imgs, - dir_flow_eval_labels, - input_height,input_width,blur_k,blur_aug, - flip_aug,binarization,scaling,scales,flip_index, - scaling_bluring,scaling_binarization,rotation, - rotation_not_90,thetha,scaling_flip, - augmentation=False,patches=patches) - + input_height, input_width, blur_k, blur_aug, + flip_aug, binarization, scaling, scales, flip_index, + scaling_bluring, scaling_binarization, rotation, + rotation_not_90, thetha, scaling_flip, + augmentation=augmentation, patches=patches) + + provide_patches(dir_img_val, dir_seg_val, dir_flow_eval_imgs, + dir_flow_eval_labels, + input_height, input_width, blur_k, blur_aug, + flip_aug, binarization, scaling, scales, flip_index, + scaling_bluring, scaling_binarization, rotation, + rotation_not_90, thetha, scaling_flip, + augmentation=False, patches=patches) - if weighted_loss: - weights=np.zeros(n_classes) + weights = np.zeros(n_classes) if data_is_provided: for obj in os.listdir(dir_flow_train_labels): try: - label_obj=cv2.imread(dir_flow_train_labels+'/'+obj) - label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) - weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + label_obj = cv2.imread(dir_flow_train_labels + '/' + obj) + label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) + weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) except: pass else: - + for obj in os.listdir(dir_seg): try: - label_obj=cv2.imread(dir_seg+'/'+obj) - label_obj_one_hot=get_one_hot( label_obj,label_obj.shape[0],label_obj.shape[1],n_classes) - weights+=(label_obj_one_hot.sum(axis=0)).sum(axis=0) + label_obj = cv2.imread(dir_seg + '/' + obj) + label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) + weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) except: pass - - weights=1.00/weights - - weights=weights/float(np.sum(weights)) - weights=weights/float(np.min(weights)) - weights=weights/float(np.sum(weights)) - - - + weights = 1.00 / weights + + weights = weights / float(np.sum(weights)) + weights = weights / float(np.min(weights)) + weights = weights / float(np.sum(weights)) + if continue_training: if is_loss_soft_dice: - model = load_model (dir_of_start_model, compile = True, custom_objects={'soft_dice_loss': soft_dice_loss}) + model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) if weighted_loss: - model = load_model (dir_of_start_model, compile = True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + model = load_model(dir_of_start_model, compile=True, + custom_objects={'loss': weighted_categorical_crossentropy(weights)}) if not is_loss_soft_dice and not weighted_loss: - model = load_model (dir_of_start_model, compile = True) + model = load_model(dir_of_start_model, compile=True) else: - #get our model. + # get our model. index_start = 0 - model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) - - #if you want to see the model structure just uncomment model summary. - #model.summary() - + model = resnet50_unet(n_classes, input_height, input_width, weight_decay, pretraining) + + # if you want to see the model structure just uncomment model summary. + # model.summary() if not is_loss_soft_dice and not weighted_loss: model.compile(loss='categorical_crossentropy', - optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - if is_loss_soft_dice: + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if is_loss_soft_dice: model.compile(loss=soft_dice_loss, - optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if weighted_loss: model.compile(loss=weighted_categorical_crossentropy(weights), - optimizer = Adam(lr=learning_rate),metrics=['accuracy']) - - #generating train and evaluation data - train_gen = data_gen(dir_flow_train_imgs,dir_flow_train_labels, batch_size = n_batch, - input_height=input_height, input_width=input_width,n_classes=n_classes ) - val_gen = data_gen(dir_flow_eval_imgs,dir_flow_eval_labels, batch_size = n_batch, - input_height=input_height, input_width=input_width,n_classes=n_classes ) - - for i in tqdm(range(index_start, n_epochs+index_start)): + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + + # generating train and evaluation data + train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, batch_size=n_batch, + input_height=input_height, input_width=input_width, n_classes=n_classes) + val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, batch_size=n_batch, + input_height=input_height, input_width=input_width, n_classes=n_classes) + + for i in tqdm(range(index_start, n_epochs + index_start)): model.fit_generator( train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs))/n_batch)-1, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, validation_data=val_gen, validation_steps=1, epochs=1) - model.save(dir_output+'/'+'model_'+str(i)) - - - #os.system('rm -rf '+dir_train_flowing) - #os.system('rm -rf '+dir_eval_flowing) - - #model.save(dir_output+'/'+'model'+'.h5') - - - - - - - - + model.save(dir_output + '/' + 'model_' + str(i)) + # os.system('rm -rf '+dir_train_flowing) + # os.system('rm -rf '+dir_eval_flowing) + # model.save(dir_output+'/'+'model'+'.h5') diff --git a/train/utils.py b/train/utils.py index 19ab46e..7c65f18 100644 --- a/train/utils.py +++ b/train/utils.py @@ -10,18 +10,17 @@ import imutils import math - -def bluring(img_in,kind): - if kind=='guass': - img_blur = cv2.GaussianBlur(img_in,(5,5),0) - elif kind=="median": - img_blur = cv2.medianBlur(img_in,5) - elif kind=='blur': - img_blur=cv2.blur(img_in,(5,5)) +def bluring(img_in, kind): + if kind == 'gauss': + img_blur = cv2.GaussianBlur(img_in, (5, 5), 0) + elif kind == "median": + img_blur = cv2.medianBlur(img_in, 5) + elif kind == 'blur': + img_blur = cv2.blur(img_in, (5, 5)) return img_blur -def elastic_transform(image, alpha, sigma,seedj, random_state=None): - + +def elastic_transform(image, alpha, sigma, seedj, random_state=None): """Elastic deformation of images as described in [Simard2003]_. .. [Simard2003] Simard, Steinkraus and Platt, "Best Practices for Convolutional Neural Networks applied to Visual Document Analysis", in @@ -37,461 +36,459 @@ def elastic_transform(image, alpha, sigma,seedj, random_state=None): dz = np.zeros_like(dx) x, y, z = np.meshgrid(np.arange(shape[1]), np.arange(shape[0]), np.arange(shape[2])) - indices = np.reshape(y+dy, (-1, 1)), np.reshape(x+dx, (-1, 1)), np.reshape(z, (-1, 1)) + indices = np.reshape(y + dy, (-1, 1)), np.reshape(x + dx, (-1, 1)), np.reshape(z, (-1, 1)) distored_image = map_coordinates(image, indices, order=1, mode='reflect') return distored_image.reshape(image.shape) + def rotation_90(img): - img_rot=np.zeros((img.shape[1],img.shape[0],img.shape[2])) - img_rot[:,:,0]=img[:,:,0].T - img_rot[:,:,1]=img[:,:,1].T - img_rot[:,:,2]=img[:,:,2].T + img_rot = np.zeros((img.shape[1], img.shape[0], img.shape[2])) + img_rot[:, :, 0] = img[:, :, 0].T + img_rot[:, :, 1] = img[:, :, 1].T + img_rot[:, :, 2] = img[:, :, 2].T return img_rot + def rotatedRectWithMaxArea(w, h, angle): - """ + """ Given a rectangle of size wxh that has been rotated by 'angle' (in radians), computes the width and height of the largest possible axis-aligned rectangle (maximal area) within the rotated rectangle. """ - if w <= 0 or h <= 0: - return 0,0 + if w <= 0 or h <= 0: + return 0, 0 - width_is_longer = w >= h - side_long, side_short = (w,h) if width_is_longer else (h,w) + width_is_longer = w >= h + side_long, side_short = (w, h) if width_is_longer else (h, w) - # since the solutions for angle, -angle and 180-angle are all the same, - # if suffices to look at the first quadrant and the absolute values of sin,cos: - sin_a, cos_a = abs(math.sin(angle)), abs(math.cos(angle)) - if side_short <= 2.*sin_a*cos_a*side_long or abs(sin_a-cos_a) < 1e-10: - # half constrained case: two crop corners touch the longer side, - # the other two corners are on the mid-line parallel to the longer line - x = 0.5*side_short - wr,hr = (x/sin_a,x/cos_a) if width_is_longer else (x/cos_a,x/sin_a) - else: - # fully constrained case: crop touches all 4 sides - cos_2a = cos_a*cos_a - sin_a*sin_a - wr,hr = (w*cos_a - h*sin_a)/cos_2a, (h*cos_a - w*sin_a)/cos_2a + # since the solutions for angle, -angle and 180-angle are all the same, + # if suffices to look at the first quadrant and the absolute values of sin,cos: + sin_a, cos_a = abs(math.sin(angle)), abs(math.cos(angle)) + if side_short <= 2. * sin_a * cos_a * side_long or abs(sin_a - cos_a) < 1e-10: + # half constrained case: two crop corners touch the longer side, + # the other two corners are on the mid-line parallel to the longer line + x = 0.5 * side_short + wr, hr = (x / sin_a, x / cos_a) if width_is_longer else (x / cos_a, x / sin_a) + else: + # fully constrained case: crop touches all 4 sides + cos_2a = cos_a * cos_a - sin_a * sin_a + wr, hr = (w * cos_a - h * sin_a) / cos_2a, (h * cos_a - w * sin_a) / cos_2a - return wr,hr + return wr, hr -def rotate_max_area(image,rotated, rotated_label,angle): + +def rotate_max_area(image, rotated, rotated_label, angle): """ image: cv2 image matrix object angle: in degree """ wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle)) h, w, _ = rotated.shape - y1 = h//2 - int(hr/2) + y1 = h // 2 - int(hr / 2) y2 = y1 + int(hr) - x1 = w//2 - int(wr/2) + x1 = w // 2 - int(wr / 2) x2 = x1 + int(wr) - return rotated[y1:y2, x1:x2],rotated_label[y1:y2, x1:x2] -def rotation_not_90_func(img,label,thetha): - rotated=imutils.rotate(img,thetha) - rotated_label=imutils.rotate(label,thetha) - return rotate_max_area(img, rotated,rotated_label,thetha) + return rotated[y1:y2, x1:x2], rotated_label[y1:y2, x1:x2] + + +def rotation_not_90_func(img, label, thetha): + rotated = imutils.rotate(img, thetha) + rotated_label = imutils.rotate(label, thetha) + return rotate_max_area(img, rotated, rotated_label, thetha) + def color_images(seg, n_classes): - ann_u=range(n_classes) - if len(np.shape(seg))==3: - seg=seg[:,:,0] - - seg_img=np.zeros((np.shape(seg)[0],np.shape(seg)[1],3)).astype(float) - colors=sns.color_palette("hls", n_classes) - + ann_u = range(n_classes) + if len(np.shape(seg)) == 3: + seg = seg[:, :, 0] + + seg_img = np.zeros((np.shape(seg)[0], np.shape(seg)[1], 3)).astype(float) + colors = sns.color_palette("hls", n_classes) + for c in ann_u: - c=int(c) - segl=(seg==c) - seg_img[:,:,0]+=segl*(colors[c][0]) - seg_img[:,:,1]+=segl*(colors[c][1]) - seg_img[:,:,2]+=segl*(colors[c][2]) + c = int(c) + segl = (seg == c) + seg_img[:, :, 0] += segl * (colors[c][0]) + seg_img[:, :, 1] += segl * (colors[c][1]) + seg_img[:, :, 2] += segl * (colors[c][2]) return seg_img - -def resize_image(seg_in,input_height,input_width): - return cv2.resize(seg_in,(input_width,input_height),interpolation=cv2.INTER_NEAREST) -def get_one_hot(seg,input_height,input_width,n_classes): - seg=seg[:,:,0] - seg_f=np.zeros((input_height, input_width,n_classes)) + +def resize_image(seg_in, input_height, input_width): + return cv2.resize(seg_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) + + +def get_one_hot(seg, input_height, input_width, n_classes): + seg = seg[:, :, 0] + seg_f = np.zeros((input_height, input_width, n_classes)) for j in range(n_classes): - seg_f[:,:,j]=(seg==j).astype(int) + seg_f[:, :, j] = (seg == j).astype(int) return seg_f - -def IoU(Yi,y_predi): + +def IoU(Yi, y_predi): ## mean Intersection over Union ## Mean IoU = TP/(FN + TP + FP) IoUs = [] - classes_true=np.unique(Yi) + classes_true = np.unique(Yi) for c in classes_true: - TP = np.sum( (Yi == c)&(y_predi==c) ) - FP = np.sum( (Yi != c)&(y_predi==c) ) - FN = np.sum( (Yi == c)&(y_predi != c)) - IoU = TP/float(TP + FP + FN) - print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c,TP,FP,FN,IoU)) + TP = np.sum((Yi == c) & (y_predi == c)) + FP = np.sum((Yi != c) & (y_predi == c)) + FN = np.sum((Yi == c) & (y_predi != c)) + IoU = TP / float(TP + FP + FN) + print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c, TP, FP, FN, IoU)) IoUs.append(IoU) mIoU = np.mean(IoUs) print("_________________") print("Mean IoU: {:4.3f}".format(mIoU)) return mIoU -def data_gen(img_folder, mask_folder, batch_size,input_height, input_width,n_classes): + + +def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_classes): c = 0 - n = [f for f in os.listdir(img_folder) if not f.startswith('.')]# os.listdir(img_folder) #List of training images + n = [f for f in os.listdir(img_folder) if not f.startswith('.')] # os.listdir(img_folder) #List of training images random.shuffle(n) while True: img = np.zeros((batch_size, input_height, input_width, 3)).astype('float') mask = np.zeros((batch_size, input_height, input_width, n_classes)).astype('float') - - for i in range(c, c+batch_size): #initially from 0 to 16, c = 0. - #print(img_folder+'/'+n[i]) - - try: - filename=n[i].split('.')[0] - - train_img = cv2.imread(img_folder+'/'+n[i])/255. - train_img = cv2.resize(train_img, (input_width, input_height),interpolation=cv2.INTER_NEAREST)# Read an image from folder and resize - - img[i-c] = train_img #add to array - img[0], img[1], and so on. - train_mask = cv2.imread(mask_folder+'/'+filename+'.png') - #print(mask_folder+'/'+filename+'.png') - #print(train_mask.shape) - train_mask = get_one_hot( resize_image(train_mask,input_height,input_width),input_height,input_width,n_classes) - #train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] - - mask[i-c] = train_mask - except: - img[i-c] = np.ones((input_height, input_width, 3)).astype('float') - mask[i-c] = np.zeros((input_height, input_width, n_classes)).astype('float') - - - c+=batch_size - if(c+batch_size>=len(os.listdir(img_folder))): - c=0 + for i in range(c, c + batch_size): # initially from 0 to 16, c = 0. + # print(img_folder+'/'+n[i]) + + try: + filename = n[i].split('.')[0] + + train_img = cv2.imread(img_folder + '/' + n[i]) / 255. + train_img = cv2.resize(train_img, (input_width, input_height), + interpolation=cv2.INTER_NEAREST) # Read an image from folder and resize + + img[i - c] = train_img # add to array - img[0], img[1], and so on. + train_mask = cv2.imread(mask_folder + '/' + filename + '.png') + # print(mask_folder+'/'+filename+'.png') + # print(train_mask.shape) + train_mask = get_one_hot(resize_image(train_mask, input_height, input_width), input_height, input_width, + n_classes) + # train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] + + mask[i - c] = train_mask + except: + img[i - c] = np.ones((input_height, input_width, 3)).astype('float') + mask[i - c] = np.zeros((input_height, input_width, n_classes)).astype('float') + + c += batch_size + if c + batch_size >= len(os.listdir(img_folder)): + c = 0 random.shuffle(n) yield img, mask - + + def otsu_copy(img): - img_r=np.zeros(img.shape) - img1=img[:,:,0] - img2=img[:,:,1] - img3=img[:,:,2] - _, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - _, threshold2 = cv2.threshold(img2, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - _, threshold3 = cv2.threshold(img3, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - img_r[:,:,0]=threshold1 - img_r[:,:,1]=threshold1 - img_r[:,:,2]=threshold1 + img_r = np.zeros(img.shape) + img1 = img[:, :, 0] + img2 = img[:, :, 1] + img3 = img[:, :, 2] + _, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + _, threshold2 = cv2.threshold(img2, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + _, threshold3 = cv2.threshold(img3, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + img_r[:, :, 0] = threshold1 + img_r[:, :, 1] = threshold1 + img_r[:, :, 2] = threshold1 return img_r -def get_patches(dir_img_f,dir_seg_f,img,label,height,width,indexer): - if img.shape[0]int(nxf): - nxf=int(nxf)+1 - if nyf>int(nyf): - nyf=int(nyf)+1 - - nxf=int(nxf) - nyf=int(nyf) - + +def get_patches(dir_img_f, dir_seg_f, img, label, height, width, indexer): + if img.shape[0] < height or img.shape[1] < width: + img, label = do_padding(img, label, height, width) + + img_h = img.shape[0] + img_w = img.shape[1] + + nxf = img_w / float(width) + nyf = img_h / float(height) + + if nxf > int(nxf): + nxf = int(nxf) + 1 + if nyf > int(nyf): + nyf = int(nyf) + 1 + + nxf = int(nxf) + nyf = int(nyf) + for i in range(nxf): for j in range(nyf): - index_x_d=i*width - index_x_u=(i+1)*width - - index_y_d=j*height - index_y_u=(j+1)*height - - if index_x_u>img_w: - index_x_u=img_w - index_x_d=img_w-width - if index_y_u>img_h: - index_y_u=img_h - index_y_d=img_h-height - - - img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] - label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] - - cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) - cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) - indexer+=1 - - return indexer + index_x_d = i * width + index_x_u = (i + 1) * width -def do_padding(img,label,height,width): - - height_new=img.shape[0] - width_new=img.shape[1] - - h_start=0 - w_start=0 - - if img.shape[0]int(nxf): - nxf=int(nxf)+1 - if nyf>int(nyf): - nyf=int(nyf)+1 - - nxf=int(nxf) - nyf=int(nyf) - - for i in range(nxf): - for j in range(nyf): - index_x_d=i*width_scale - index_x_u=(i+1)*width_scale - - index_y_d=j*height_scale - index_y_u=(j+1)*height_scale - - if index_x_u>img_w: - index_x_u=img_w - index_x_d=img_w-width_scale - if index_y_u>img_h: - index_y_u=img_h - index_y_d=img_h-height_scale - - - img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] - label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] - - img_patch=resize_image(img_patch,height,width) - label_patch=resize_image(label_patch,height,width) - - cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) - cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) - indexer+=1 + index_y_d = j * height + index_y_u = (j + 1) * height - return indexer + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - width + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - height -def get_patches_num_scale_new(dir_img_f,dir_seg_f,img,label,height,width,indexer,scaler): - img=resize_image(img,int(img.shape[0]*scaler),int(img.shape[1]*scaler)) - label=resize_image(label,int(label.shape[0]*scaler),int(label.shape[1]*scaler)) - - if img.shape[0]int(nxf): - nxf=int(nxf)+1 - if nyf>int(nyf): - nyf=int(nyf)+1 - - nxf=int(nxf) - nyf=int(nyf) - - for i in range(nxf): - for j in range(nyf): - index_x_d=i*width_scale - index_x_u=(i+1)*width_scale - - index_y_d=j*height_scale - index_y_u=(j+1)*height_scale - - if index_x_u>img_w: - index_x_u=img_w - index_x_d=img_w-width_scale - if index_y_u>img_h: - index_y_u=img_h - index_y_d=img_h-height_scale - - - img_patch=img[index_y_d:index_y_u,index_x_d:index_x_u,:] - label_patch=label[index_y_d:index_y_u,index_x_d:index_x_u,:] - - #img_patch=resize_image(img_patch,height,width) - #label_patch=resize_image(label_patch,height,width) - - cv2.imwrite(dir_img_f+'/img_'+str(indexer)+'.png', img_patch ) - cv2.imwrite(dir_seg_f+'/img_'+str(indexer)+'.png' , label_patch ) - indexer+=1 + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + label_patch = label[index_y_d:index_y_u, index_x_d:index_x_u, :] + + cv2.imwrite(dir_img_f + '/img_' + str(indexer) + '.png', img_patch) + cv2.imwrite(dir_seg_f + '/img_' + str(indexer) + '.png', label_patch) + indexer += 1 return indexer -def provide_patches(dir_img,dir_seg,dir_flow_train_imgs, +def do_padding(img, label, height, width): + height_new = img.shape[0] + width_new = img.shape[1] + + h_start = 0 + w_start = 0 + + if img.shape[0] < height: + h_start = int(abs(height - img.shape[0]) / 2.) + height_new = height + + if img.shape[1] < width: + w_start = int(abs(width - img.shape[1]) / 2.) + width_new = width + + img_new = np.ones((height_new, width_new, img.shape[2])).astype(float) * 255 + label_new = np.zeros((height_new, width_new, label.shape[2])).astype(float) + + img_new[h_start:h_start + img.shape[0], w_start:w_start + img.shape[1], :] = np.copy(img[:, :, :]) + label_new[h_start:h_start + label.shape[0], w_start:w_start + label.shape[1], :] = np.copy(label[:, :, :]) + + return img_new, label_new + + +def get_patches_num_scale(dir_img_f, dir_seg_f, img, label, height, width, indexer, n_patches, scaler): + if img.shape[0] < height or img.shape[1] < width: + img, label = do_padding(img, label, height, width) + + img_h = img.shape[0] + img_w = img.shape[1] + + height_scale = int(height * scaler) + width_scale = int(width * scaler) + + nxf = img_w / float(width_scale) + nyf = img_h / float(height_scale) + + if nxf > int(nxf): + nxf = int(nxf) + 1 + if nyf > int(nyf): + nyf = int(nyf) + 1 + + nxf = int(nxf) + nyf = int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d = i * width_scale + index_x_u = (i + 1) * width_scale + + index_y_d = j * height_scale + index_y_u = (j + 1) * height_scale + + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - width_scale + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - height_scale + + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + label_patch = label[index_y_d:index_y_u, index_x_d:index_x_u, :] + + img_patch = resize_image(img_patch, height, width) + label_patch = resize_image(label_patch, height, width) + + cv2.imwrite(dir_img_f + '/img_' + str(indexer) + '.png', img_patch) + cv2.imwrite(dir_seg_f + '/img_' + str(indexer) + '.png', label_patch) + indexer += 1 + + return indexer + + +def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, indexer, scaler): + img = resize_image(img, int(img.shape[0] * scaler), int(img.shape[1] * scaler)) + label = resize_image(label, int(label.shape[0] * scaler), int(label.shape[1] * scaler)) + + if img.shape[0] < height or img.shape[1] < width: + img, label = do_padding(img, label, height, width) + + img_h = img.shape[0] + img_w = img.shape[1] + + height_scale = int(height * 1) + width_scale = int(width * 1) + + nxf = img_w / float(width_scale) + nyf = img_h / float(height_scale) + + if nxf > int(nxf): + nxf = int(nxf) + 1 + if nyf > int(nyf): + nyf = int(nyf) + 1 + + nxf = int(nxf) + nyf = int(nyf) + + for i in range(nxf): + for j in range(nyf): + index_x_d = i * width_scale + index_x_u = (i + 1) * width_scale + + index_y_d = j * height_scale + index_y_u = (j + 1) * height_scale + + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - width_scale + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - height_scale + + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + label_patch = label[index_y_d:index_y_u, index_x_d:index_x_u, :] + + # img_patch=resize_image(img_patch,height,width) + # label_patch=resize_image(label_patch,height,width) + + cv2.imwrite(dir_img_f + '/img_' + str(indexer) + '.png', img_patch) + cv2.imwrite(dir_seg_f + '/img_' + str(indexer) + '.png', label_patch) + indexer += 1 + + return indexer + + +def provide_patches(dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, - input_height,input_width,blur_k,blur_aug, - flip_aug,binarization,scaling,scales,flip_index, - scaling_bluring,scaling_binarization,rotation, - rotation_not_90,thetha,scaling_flip, - augmentation=False,patches=False): - - imgs_cv_train=np.array(os.listdir(dir_img)) - segs_cv_train=np.array(os.listdir(dir_seg)) - - indexer=0 - for im, seg_i in tqdm(zip(imgs_cv_train,segs_cv_train)): - img_name=im.split('.')[0] + input_height, input_width, blur_k, blur_aug, + flip_aug, binarization, scaling, scales, flip_index, + scaling_bluring, scaling_binarization, rotation, + rotation_not_90, thetha, scaling_flip, + augmentation=False, patches=False): + imgs_cv_train = np.array(os.listdir(dir_img)) + segs_cv_train = np.array(os.listdir(dir_seg)) + + indexer = 0 + for im, seg_i in tqdm(zip(imgs_cv_train, segs_cv_train)): + img_name = im.split('.')[0] if not patches: - cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', resize_image(cv2.imread(dir_img+'/'+im),input_height,input_width ) ) - cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width ) ) - indexer+=1 - + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_img + '/' + im), input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) + indexer += 1 + if augmentation: if flip_aug: for f_i in flip_index: - cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', - resize_image(cv2.flip(cv2.imread(dir_img+'/'+im),f_i),input_height,input_width) ) - - cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , - resize_image(cv2.flip(cv2.imread(dir_seg+'/'+img_name+'.png'),f_i),input_height,input_width) ) - indexer+=1 - - if blur_aug: - for blur_i in blur_k: - cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', - (resize_image(bluring(cv2.imread(dir_img+'/'+im),blur_i),input_height,input_width) ) ) - - cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png' , - resize_image(cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width) ) - indexer+=1 - - - if binarization: - cv2.imwrite(dir_flow_train_imgs+'/img_'+str(indexer)+'.png', - resize_image(otsu_copy( cv2.imread(dir_img+'/'+im)),input_height,input_width )) - - cv2.imwrite(dir_flow_train_labels+'/img_'+str(indexer)+'.png', - resize_image( cv2.imread(dir_seg+'/'+img_name+'.png'),input_height,input_width )) - indexer+=1 - - - + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(cv2.flip(cv2.imread(dir_img + '/' + im), f_i), input_height, + input_width)) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), + input_height, input_width)) + indexer += 1 + + if blur_aug: + for blur_i in blur_k: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(bluring(cv2.imread(dir_img + '/' + im), blur_i), input_height, + input_width))) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, + input_width)) + indexer += 1 + + if binarization: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(otsu_copy(cv2.imread(dir_img + '/' + im)), input_height, input_width)) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) + indexer += 1 - - if patches: - - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer) - + + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + cv2.imread(dir_img + '/' + im), cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer) + if augmentation: - + if rotation: - - - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - rotation_90( cv2.imread(dir_img+'/'+im) ), - rotation_90( cv2.imread(dir_seg+'/'+img_name+'.png') ), - input_height,input_width,indexer=indexer) - + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + rotation_90(cv2.imread(dir_img + '/' + im)), + rotation_90(cv2.imread(dir_seg + '/' + img_name + '.png')), + input_height, input_width, indexer=indexer) + if rotation_not_90: - + for thetha_i in thetha: - img_max_rotated,label_max_rotated=rotation_not_90_func(cv2.imread(dir_img+'/'+im),cv2.imread(dir_seg+'/'+img_name+'.png'),thetha_i) - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - img_max_rotated, - label_max_rotated, - input_height,input_width,indexer=indexer) + img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/' + im), + cv2.imread( + dir_seg + '/' + img_name + '.png'), + thetha_i) + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + img_max_rotated, + label_max_rotated, + input_height, input_width, indexer=indexer) if flip_aug: for f_i in flip_index: - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - cv2.flip( cv2.imread(dir_img+'/'+im) , f_i), - cv2.flip( cv2.imread(dir_seg+'/'+img_name+'.png') ,f_i), - input_height,input_width,indexer=indexer) - if blur_aug: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + cv2.flip(cv2.imread(dir_img + '/' + im), f_i), + cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), + input_height, input_width, indexer=indexer) + if blur_aug: for blur_i in blur_k: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + bluring(cv2.imread(dir_img + '/' + im), blur_i), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer) - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - bluring( cv2.imread(dir_img+'/'+im) , blur_i), - cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer) - - - if scaling: + if scaling: for sc_ind in scales: - indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, - cv2.imread(dir_img+'/'+im) , - cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer,scaler=sc_ind) + indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, + cv2.imread(dir_img + '/' + im), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer, scaler=sc_ind) if binarization: - indexer=get_patches(dir_flow_train_imgs,dir_flow_train_labels, - otsu_copy( cv2.imread(dir_img+'/'+im)), - cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer) - - - - if scaling_bluring: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + otsu_copy(cv2.imread(dir_img + '/' + im)), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer) + + if scaling_bluring: for sc_ind in scales: for blur_i in blur_k: - indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, - bluring( cv2.imread(dir_img+'/'+im) , blur_i) , - cv2.imread(dir_seg+'/'+img_name+'.png') , - input_height,input_width,indexer=indexer,scaler=sc_ind) + indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, + bluring(cv2.imread(dir_img + '/' + im), blur_i), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer, + scaler=sc_ind) - if scaling_binarization: + if scaling_binarization: for sc_ind in scales: - indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, - otsu_copy( cv2.imread(dir_img+'/'+im)) , - cv2.imread(dir_seg+'/'+img_name+'.png'), - input_height,input_width,indexer=indexer,scaler=sc_ind) - - if scaling_flip: + indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, + otsu_copy(cv2.imread(dir_img + '/' + im)), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer, scaler=sc_ind) + + if scaling_flip: for sc_ind in scales: for f_i in flip_index: - indexer=get_patches_num_scale_new(dir_flow_train_imgs,dir_flow_train_labels, - cv2.flip( cv2.imread(dir_img+'/'+im) , f_i) , - cv2.flip(cv2.imread(dir_seg+'/'+img_name+'.png') ,f_i) , - input_height,input_width,indexer=indexer,scaler=sc_ind) - - - - - - - + indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, + cv2.flip(cv2.imread(dir_img + '/' + im), f_i), + cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), + f_i), + input_height, input_width, indexer=indexer, + scaler=sc_ind) From 6e06742e66be00aba83919a3d49774ed1f54c790 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 16 Apr 2024 01:00:48 +0200 Subject: [PATCH 039/374] first working update of branch --- train/config_params.json | 19 ++- train/models.py | 179 +++++++++++++++++++++++++ train/train.py | 132 ++++++++++++------- train/utils.py | 273 +++++++++++++++++++++++++-------------- 4 files changed, 452 insertions(+), 151 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index 7505a81..bd47a52 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,8 +1,9 @@ { - "n_classes" : 3, + "model_name" : "hybrid_transformer_cnn", + "n_classes" : 2, "n_epochs" : 2, "input_height" : 448, - "input_width" : 672, + "input_width" : 448, "weight_decay" : 1e-6, "n_batch" : 2, "learning_rate": 1e-4, @@ -18,13 +19,21 @@ "scaling_flip" : false, "rotation": false, "rotation_not_90": false, + "num_patches_xy": [28, 28], + "transformer_patchsize": 1, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], "continue_training": false, - "index_start": 0, - "dir_of_start_model": " ", + "index_start" : 0, + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, "dir_train": "/train", "dir_eval": "/eval", - "dir_output": "/output" + "dir_output": "/out" } diff --git a/train/models.py b/train/models.py index f06823e..f7a7ad8 100644 --- a/train/models.py +++ b/train/models.py @@ -1,13 +1,81 @@ +import tensorflow as tf +from tensorflow import keras from tensorflow.keras.models import * from tensorflow.keras.layers import * from tensorflow.keras import layers from tensorflow.keras.regularizers import l2 +mlp_head_units = [2048, 1024] +projection_dim = 64 +transformer_layers = 8 +num_heads = 4 resnet50_Weights_path = './pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' IMAGE_ORDERING = 'channels_last' MERGE_AXIS = -1 +transformer_units = [ + projection_dim * 2, + projection_dim, +] # Size of the transformer layers +def mlp(x, hidden_units, dropout_rate): + for units in hidden_units: + x = layers.Dense(units, activation=tf.nn.gelu)(x) + x = layers.Dropout(dropout_rate)(x) + return x + +class Patches(layers.Layer): + def __init__(self, patch_size):#__init__(self, **kwargs):#:__init__(self, patch_size):#__init__(self, **kwargs): + super(Patches, self).__init__() + self.patch_size = patch_size + + def call(self, images): + print(tf.shape(images)[1],'images') + print(self.patch_size,'self.patch_size') + batch_size = tf.shape(images)[0] + patches = tf.image.extract_patches( + images=images, + sizes=[1, self.patch_size, self.patch_size, 1], + strides=[1, self.patch_size, self.patch_size, 1], + rates=[1, 1, 1, 1], + padding="VALID", + ) + patch_dims = patches.shape[-1] + print(patches.shape,patch_dims,'patch_dims') + patches = tf.reshape(patches, [batch_size, -1, patch_dims]) + return patches + def get_config(self): + + config = super().get_config().copy() + config.update({ + 'patch_size': self.patch_size, + }) + return config + +class PatchEncoder(layers.Layer): + def __init__(self, num_patches, projection_dim): + super(PatchEncoder, self).__init__() + self.num_patches = num_patches + self.projection = layers.Dense(units=projection_dim) + self.position_embedding = layers.Embedding( + input_dim=num_patches, output_dim=projection_dim + ) + + def call(self, patch): + positions = tf.range(start=0, limit=self.num_patches, delta=1) + encoded = self.projection(patch) + self.position_embedding(positions) + return encoded + def get_config(self): + + config = super().get_config().copy() + config.update({ + 'num_patches': self.num_patches, + 'projection': self.projection, + 'position_embedding': self.position_embedding, + }) + return config + + def one_side_pad(x): x = ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING)(x) if IMAGE_ORDERING == 'channels_first': @@ -292,3 +360,114 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, weight_decay=1e- model = Model(img_input, o) return model + + +def vit_resnet50_unet(n_classes,patch_size, num_patches, input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + inputs = layers.Input(shape=(input_height, input_width, 3)) + IMAGE_ORDERING = 'channels_last' + bn_axis=3 + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(inputs) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3), data_format=IMAGE_ORDERING, strides=(2, 2))(x) + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x) + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + if pretraining: + model = keras.Model(inputs, x).load_weights(resnet50_Weights_path) + + num_patches = x.shape[1]*x.shape[2] + patches = Patches(patch_size)(x) + # Encode patches. + encoded_patches = PatchEncoder(num_patches, projection_dim)(patches) + + for _ in range(transformer_layers): + # Layer normalization 1. + x1 = layers.LayerNormalization(epsilon=1e-6)(encoded_patches) + # Create a multi-head attention layer. + attention_output = layers.MultiHeadAttention( + num_heads=num_heads, key_dim=projection_dim, dropout=0.1 + )(x1, x1) + # Skip connection 1. + x2 = layers.Add()([attention_output, encoded_patches]) + # Layer normalization 2. + x3 = layers.LayerNormalization(epsilon=1e-6)(x2) + # MLP. + x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1) + # Skip connection 2. + encoded_patches = layers.Add()([x3, x2]) + + encoded_patches = tf.reshape(encoded_patches, [-1, x.shape[1], x.shape[2], 64]) + + v1024_2048 = Conv2D( 1024 , (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(encoded_patches) + v1024_2048 = (BatchNormalization(axis=bn_axis))(v1024_2048) + v1024_2048 = Activation('relu')(v1024_2048) + + o = (UpSampling2D( (2, 2), data_format=IMAGE_ORDERING))(v1024_2048) + o = (concatenate([o, f4],axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o ,f3], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f2], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(128, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f1], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(64, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, inputs],axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(32, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + + model = keras.Model(inputs=inputs, outputs=o) + + return model diff --git a/train/train.py b/train/train.py index 03faf46..6e6a172 100644 --- a/train/train.py +++ b/train/train.py @@ -10,6 +10,7 @@ from utils import * from metrics import * from tensorflow.keras.models import load_model from tqdm import tqdm +import json def configuration(): @@ -42,9 +43,13 @@ def config_params(): learning_rate = 1e-4 # Set the learning rate. patches = False # Divides input image into smaller patches (input size of the model) when set to true. For the model to see the full image, like page extraction, set this to false. augmentation = False # To apply any kind of augmentation, this parameter must be set to true. - flip_aug = False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in train.py. - blur_aug = False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in train.py. - scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in train.py. + flip_aug = False # If true, different types of flipping will be applied to the image. Types of flips are defined with "flip_index" in config_params.json. + blur_aug = False # If true, different types of blurring will be applied to the image. Types of blur are defined with "blur_k" in config_params.json. + padding_white = False # If true, white padding will be applied to the image. + padding_black = False # If true, black padding will be applied to the image. + scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in config_params.json. + degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json. + brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". @@ -52,13 +57,18 @@ def config_params(): pretraining = False # Set to true to load pretrained weights of ResNet50 encoder. scaling_bluring = False # If true, a combination of scaling and blurring will be applied to the image. scaling_binarization = False # If true, a combination of scaling and binarization will be applied to the image. + scaling_brightness = False # If true, a combination of scaling and brightening will be applied to the image. scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image. - thetha = [10, -10] # Rotate image by these angles for augmentation. - blur_k = ['blur', 'gauss', 'median'] # Blur image for augmentation. - scales = [0.5, 2] # Scale patches for augmentation. - flip_index = [0, 1, -1] # Flip image for augmentation. + thetha = None # Rotate image by these angles for augmentation. + blur_k = None # Blur image for augmentation. + scales = None # Scale patches for augmentation. + degrade_scales = None # Degrade image for augmentation. + brightness = None # Brighten image for augmentation. + flip_index = None # Flip image for augmentation. continue_training = False # Set to true if you would like to continue training an already trained a model. - index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. + transformer_patchsize = None # Patch size of vision transformer patches. + num_patches_xy = None # Number of patches for vision transformer. + index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. @@ -66,15 +76,19 @@ def config_params(): @ex.automain -def run(n_classes, n_epochs, input_height, +def run(_config, n_classes, n_epochs, input_height, input_width, weight_decay, weighted_loss, index_start, dir_of_start_model, is_loss_soft_dice, n_batch, patches, augmentation, flip_aug, - blur_aug, scaling, binarization, - blur_k, scales, dir_train, data_is_provided, - scaling_bluring, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, continue_training, - flip_index, dir_eval, dir_output, pretraining, learning_rate): + blur_aug, padding_white, padding_black, scaling, degrading, + brightening, binarization, blur_k, scales, degrade_scales, + brightness, dir_train, data_is_provided, scaling_bluring, + scaling_brightness, scaling_binarization, rotation, rotation_not_90, + thetha, scaling_flip, continue_training, transformer_patchsize, + num_patches_xy, model_name, flip_index, dir_eval, dir_output, + pretraining, learning_rate): + + num_patches = num_patches_xy[0]*num_patches_xy[1] if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') dir_eval_flowing = os.path.join(dir_output, 'eval') @@ -121,23 +135,28 @@ def run(n_classes, n_epochs, input_height, # set the gpu configuration configuration() + + imgs_list=np.array(os.listdir(dir_img)) + segs_list=np.array(os.listdir(dir_seg)) + + imgs_list_test=np.array(os.listdir(dir_img_val)) + segs_list_test=np.array(os.listdir(dir_seg_val)) # writing patches into a sub-folder in order to be flowed from directory. - provide_patches(dir_img, dir_seg, dir_flow_train_imgs, - dir_flow_train_labels, - input_height, input_width, blur_k, blur_aug, - flip_aug, binarization, scaling, scales, flip_index, - scaling_bluring, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, - augmentation=augmentation, patches=patches) - - provide_patches(dir_img_val, dir_seg_val, dir_flow_eval_imgs, - dir_flow_eval_labels, - input_height, input_width, blur_k, blur_aug, - flip_aug, binarization, scaling, scales, flip_index, - scaling_bluring, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, - augmentation=False, patches=patches) + provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, + dir_flow_train_labels, input_height, input_width, blur_k, + blur_aug, padding_white, padding_black, flip_aug, binarization, + scaling, degrading, brightening, scales, degrade_scales, brightness, + flip_index, scaling_bluring, scaling_brightness, scaling_binarization, + rotation, rotation_not_90, thetha, scaling_flip, augmentation=augmentation, + patches=patches) + + provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, + dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, + blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, + scaling, degrading, brightening, scales, degrade_scales, brightness, + flip_index, scaling_bluring, scaling_brightness, scaling_binarization, + rotation, rotation_not_90, thetha, scaling_flip, augmentation=False, patches=patches) if weighted_loss: weights = np.zeros(n_classes) @@ -166,38 +185,50 @@ def run(n_classes, n_epochs, input_height, weights = weights / float(np.sum(weights)) if continue_training: - if is_loss_soft_dice: - model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) - if weighted_loss: - model = load_model(dir_of_start_model, compile=True, - custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - if not is_loss_soft_dice and not weighted_loss: - model = load_model(dir_of_start_model, compile=True) + if model_name=='resnet50_unet': + if is_loss_soft_dice: + model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) + if weighted_loss: + model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + if not is_loss_soft_dice and not weighted_loss: + model = load_model(dir_of_start_model , compile=True) + elif model_name=='hybrid_transformer_cnn': + if is_loss_soft_dice: + model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) + if weighted_loss: + model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + if not is_loss_soft_dice and not weighted_loss: + model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) else: - # get our model. index_start = 0 - model = resnet50_unet(n_classes, input_height, input_width, weight_decay, pretraining) - - # if you want to see the model structure just uncomment model summary. - # model.summary() + if model_name=='resnet50_unet': + model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + elif model_name=='hybrid_transformer_cnn': + model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width,weight_decay,pretraining) + + #if you want to see the model structure just uncomment model summary. + #model.summary() + if not is_loss_soft_dice and not weighted_loss: model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - if is_loss_soft_dice: + if is_loss_soft_dice: model.compile(loss=soft_dice_loss, optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - if weighted_loss: model.compile(loss=weighted_categorical_crossentropy(weights), optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - + # generating train and evaluation data train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, batch_size=n_batch, input_height=input_height, input_width=input_width, n_classes=n_classes) val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, batch_size=n_batch, input_height=input_height, input_width=input_width, n_classes=n_classes) - + + ##img_validation_patches = os.listdir(dir_flow_eval_imgs) + ##score_best=[] + ##score_best.append(0) for i in tqdm(range(index_start, n_epochs + index_start)): model.fit_generator( train_gen, @@ -205,9 +236,12 @@ def run(n_classes, n_epochs, input_height, validation_data=val_gen, validation_steps=1, epochs=1) - model.save(dir_output + '/' + 'model_' + str(i)) + model.save(dir_output+'/'+'model_'+str(i)) + + with open(dir_output+'/'+'model_'+str(i)+'/'+"config.json", "w") as fp: + json.dump(_config, fp) # encode dict into JSON - # os.system('rm -rf '+dir_train_flowing) - # os.system('rm -rf '+dir_eval_flowing) + #os.system('rm -rf '+dir_train_flowing) + #os.system('rm -rf '+dir_eval_flowing) - # model.save(dir_output+'/'+'model'+'.h5') + #model.save(dir_output+'/'+'model'+'.h5') diff --git a/train/utils.py b/train/utils.py index 7c65f18..c2786ec 100644 --- a/train/utils.py +++ b/train/utils.py @@ -9,6 +9,15 @@ from tqdm import tqdm import imutils import math +def do_brightening(img_in_dir, factor): + im = Image.open(img_in_dir) + enhancer = ImageEnhance.Brightness(im) + out_img = enhancer.enhance(factor) + out_img = out_img.convert('RGB') + opencv_img = np.array(out_img) + opencv_img = opencv_img[:,:,::-1].copy() + return opencv_img + def bluring(img_in, kind): if kind == 'gauss': @@ -138,11 +147,11 @@ def IoU(Yi, y_predi): FP = np.sum((Yi != c) & (y_predi == c)) FN = np.sum((Yi == c) & (y_predi != c)) IoU = TP / float(TP + FP + FN) - print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c, TP, FP, FN, IoU)) + #print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c, TP, FP, FN, IoU)) IoUs.append(IoU) mIoU = np.mean(IoUs) - print("_________________") - print("Mean IoU: {:4.3f}".format(mIoU)) + #print("_________________") + #print("Mean IoU: {:4.3f}".format(mIoU)) return mIoU @@ -241,124 +250,170 @@ def get_patches(dir_img_f, dir_seg_f, img, label, height, width, indexer): return indexer -def do_padding(img, label, height, width): - height_new = img.shape[0] - width_new = img.shape[1] +def do_padding_white(img): + img_org_h = img.shape[0] + img_org_w = img.shape[1] + + index_start_h = 4 + index_start_w = 4 + + img_padded = np.zeros((img.shape[0] + 2*index_start_h, img.shape[1]+ 2*index_start_w, img.shape[2])) + 255 + img_padded[index_start_h: index_start_h + img.shape[0], index_start_w: index_start_w + img.shape[1], :] = img[:, :, :] + + return img_padded.astype(float) + +def do_degrading(img, scale): + img_org_h = img.shape[0] + img_org_w = img.shape[1] + + img_res = resize_image(img, int(img_org_h * scale), int(img_org_w * scale)) + + return resize_image(img_res, img_org_h, img_org_w) + + +def do_padding_black(img): + img_org_h = img.shape[0] + img_org_w = img.shape[1] + + index_start_h = 4 + index_start_w = 4 + + img_padded = np.zeros((img.shape[0] + 2*index_start_h, img.shape[1] + 2*index_start_w, img.shape[2])) + img_padded[index_start_h: index_start_h + img.shape[0], index_start_w: index_start_w + img.shape[1], :] = img[:, :, :] + + return img_padded.astype(float) + + +def do_padding_label(img): + img_org_h = img.shape[0] + img_org_w = img.shape[1] + + index_start_h = 4 + index_start_w = 4 + + img_padded = np.zeros((img.shape[0] + 2*index_start_h, img.shape[1] + 2*index_start_w, img.shape[2])) + img_padded[index_start_h: index_start_h + img.shape[0], index_start_w: index_start_w + img.shape[1], :] = img[:, :, :] + + return img_padded.astype(np.int16) + +def do_padding(img, label, height, width): + height_new=img.shape[0] + width_new=img.shape[1] + h_start = 0 w_start = 0 - + if img.shape[0] < height: h_start = int(abs(height - img.shape[0]) / 2.) height_new = height - + if img.shape[1] < width: w_start = int(abs(width - img.shape[1]) / 2.) width_new = width - + img_new = np.ones((height_new, width_new, img.shape[2])).astype(float) * 255 label_new = np.zeros((height_new, width_new, label.shape[2])).astype(float) - + img_new[h_start:h_start + img.shape[0], w_start:w_start + img.shape[1], :] = np.copy(img[:, :, :]) label_new[h_start:h_start + label.shape[0], w_start:w_start + label.shape[1], :] = np.copy(label[:, :, :]) - - return img_new, label_new + + return img_new,label_new def get_patches_num_scale(dir_img_f, dir_seg_f, img, label, height, width, indexer, n_patches, scaler): if img.shape[0] < height or img.shape[1] < width: img, label = do_padding(img, label, height, width) - + img_h = img.shape[0] img_w = img.shape[1] - + height_scale = int(height * scaler) width_scale = int(width * scaler) - + + nxf = img_w / float(width_scale) nyf = img_h / float(height_scale) - + if nxf > int(nxf): nxf = int(nxf) + 1 if nyf > int(nyf): nyf = int(nyf) + 1 - + nxf = int(nxf) nyf = int(nyf) - + for i in range(nxf): for j in range(nyf): index_x_d = i * width_scale index_x_u = (i + 1) * width_scale - + index_y_d = j * height_scale index_y_u = (j + 1) * height_scale - + if index_x_u > img_w: index_x_u = img_w index_x_d = img_w - width_scale if index_y_u > img_h: index_y_u = img_h index_y_d = img_h - height_scale - + + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] label_patch = label[index_y_d:index_y_u, index_x_d:index_x_u, :] - + img_patch = resize_image(img_patch, height, width) label_patch = resize_image(label_patch, height, width) - + cv2.imwrite(dir_img_f + '/img_' + str(indexer) + '.png', img_patch) cv2.imwrite(dir_seg_f + '/img_' + str(indexer) + '.png', label_patch) indexer += 1 - + return indexer def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, indexer, scaler): img = resize_image(img, int(img.shape[0] * scaler), int(img.shape[1] * scaler)) label = resize_image(label, int(label.shape[0] * scaler), int(label.shape[1] * scaler)) - + if img.shape[0] < height or img.shape[1] < width: img, label = do_padding(img, label, height, width) - + img_h = img.shape[0] img_w = img.shape[1] - + height_scale = int(height * 1) width_scale = int(width * 1) - + nxf = img_w / float(width_scale) nyf = img_h / float(height_scale) - + if nxf > int(nxf): nxf = int(nxf) + 1 if nyf > int(nyf): nyf = int(nyf) + 1 - + nxf = int(nxf) nyf = int(nyf) - + for i in range(nxf): for j in range(nyf): index_x_d = i * width_scale index_x_u = (i + 1) * width_scale - + index_y_d = j * height_scale index_y_u = (j + 1) * height_scale - + if index_x_u > img_w: index_x_u = img_w index_x_d = img_w - width_scale if index_y_u > img_h: index_y_u = img_h index_y_d = img_h - height_scale - + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] label_patch = label[index_y_d:index_y_u, index_x_d:index_x_u, :] - - # img_patch=resize_image(img_patch,height,width) - # label_patch=resize_image(label_patch,height,width) - + cv2.imwrite(dir_img_f + '/img_' + str(indexer) + '.png', img_patch) cv2.imwrite(dir_seg_f + '/img_' + str(indexer) + '.png', label_patch) indexer += 1 @@ -366,78 +421,65 @@ def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, i return indexer -def provide_patches(dir_img, dir_seg, dir_flow_train_imgs, - dir_flow_train_labels, - input_height, input_width, blur_k, blur_aug, - flip_aug, binarization, scaling, scales, flip_index, - scaling_bluring, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, - augmentation=False, patches=False): - imgs_cv_train = np.array(os.listdir(dir_img)) - segs_cv_train = np.array(os.listdir(dir_seg)) - +def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow_train_imgs, + dir_flow_train_labels, input_height, input_width, blur_k, blur_aug, + padding_white, padding_black, flip_aug, binarization, scaling, degrading, + brightening, scales, degrade_scales, brightness, flip_index, + scaling_bluring, scaling_brightness, scaling_binarization, rotation, + rotation_not_90, thetha, scaling_flip, augmentation=False, patches=False): + indexer = 0 - for im, seg_i in tqdm(zip(imgs_cv_train, segs_cv_train)): + for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): img_name = im.split('.')[0] if not patches: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_img + '/' + im), input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_img + '/' + im), input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) indexer += 1 - + if augmentation: if flip_aug: for f_i in flip_index: cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - resize_image(cv2.flip(cv2.imread(dir_img + '/' + im), f_i), input_height, - input_width)) - + resize_image(cv2.flip(cv2.imread(dir_img+'/'+im),f_i),input_height,input_width) ) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), - input_height, input_width)) + resize_image(cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), input_height, input_width)) indexer += 1 - - if blur_aug: + + if blur_aug: for blur_i in blur_k: cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - (resize_image(bluring(cv2.imread(dir_img + '/' + im), blur_i), input_height, - input_width))) - + (resize_image(bluring(cv2.imread(dir_img + '/' + im), blur_i), input_height, input_width))) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, - input_width)) + resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) indexer += 1 - + if binarization: cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(otsu_copy(cv2.imread(dir_img + '/' + im)), input_height, input_width)) - + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) indexer += 1 - + + if patches: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, cv2.imread(dir_img + '/' + im), cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width, indexer=indexer) - + if augmentation: - if rotation: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - rotation_90(cv2.imread(dir_img + '/' + im)), - rotation_90(cv2.imread(dir_seg + '/' + img_name + '.png')), - input_height, input_width, indexer=indexer) - + rotation_90(cv2.imread(dir_img + '/' + im)), + rotation_90(cv2.imread(dir_seg + '/' + img_name + '.png')), + input_height, input_width, indexer=indexer) + if rotation_not_90: - for thetha_i in thetha: - img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/' + im), - cv2.imread( - dir_seg + '/' + img_name + '.png'), - thetha_i) + img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/'+im), + cv2.imread(dir_seg + '/'+img_name + '.png'), thetha_i) indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, img_max_rotated, label_max_rotated, @@ -448,47 +490,84 @@ def provide_patches(dir_img, dir_seg, dir_flow_train_imgs, cv2.flip(cv2.imread(dir_img + '/' + im), f_i), cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), input_height, input_width, indexer=indexer) - if blur_aug: + if blur_aug: for blur_i in blur_k: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, bluring(cv2.imread(dir_img + '/' + im), blur_i), cv2.imread(dir_seg + '/' + img_name + '.png'), - input_height, input_width, indexer=indexer) - - if scaling: + input_height, input_width, indexer=indexer) + if padding_black: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + do_padding_black(cv2.imread(dir_img + '/' + im)), + do_padding_label(cv2.imread(dir_seg + '/' + img_name + '.png')), + input_height, input_width, indexer=indexer) + + if padding_white: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + do_padding_white(cv2.imread(dir_img + '/'+im)), + do_padding_label(cv2.imread(dir_seg + '/' + img_name + '.png')), + input_height, input_width, indexer=indexer) + + if brightening: + for factor in brightness: + try: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + do_brightening(dir_img + '/' +im, factor), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer) + except: + pass + if scaling: for sc_ind in scales: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, - cv2.imread(dir_img + '/' + im), + cv2.imread(dir_img + '/' + im) , cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width, indexer=indexer, scaler=sc_ind) + + if degrading: + for degrade_scale_ind in degrade_scales: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + do_degrading(cv2.imread(dir_img + '/' + im), degrade_scale_ind), + cv2.imread(dir_seg + '/' + img_name + '.png'), + input_height, input_width, indexer=indexer) + if binarization: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, otsu_copy(cv2.imread(dir_img + '/' + im)), cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width, indexer=indexer) - if scaling_bluring: + if scaling_brightness: + for sc_ind in scales: + for factor in brightness: + try: + indexer = get_patches_num_scale_new(dir_flow_train_imgs, + dir_flow_train_labels, + do_brightening(dir_img + '/' + im, factor) + ,cv2.imread(dir_seg + '/' + img_name + '.png') + ,input_height, input_width, indexer=indexer, scaler=sc_ind) + except: + pass + + if scaling_bluring: for sc_ind in scales: for blur_i in blur_k: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, bluring(cv2.imread(dir_img + '/' + im), blur_i), cv2.imread(dir_seg + '/' + img_name + '.png'), - input_height, input_width, indexer=indexer, - scaler=sc_ind) + input_height, input_width, indexer=indexer, scaler=sc_ind) - if scaling_binarization: + if scaling_binarization: for sc_ind in scales: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, otsu_copy(cv2.imread(dir_img + '/' + im)), cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width, indexer=indexer, scaler=sc_ind) - - if scaling_flip: + + if scaling_flip: for sc_ind in scales: for f_i in flip_index: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, - cv2.flip(cv2.imread(dir_img + '/' + im), f_i), - cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), - f_i), - input_height, input_width, indexer=indexer, - scaler=sc_ind) + cv2.flip( cv2.imread(dir_img + '/' + im), f_i), + cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), + input_height, input_width, indexer=indexer, scaler=sc_ind) From ca63c097c3c30b58513d708f476139c590ac2d94 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 29 Apr 2024 20:59:36 +0200 Subject: [PATCH 040/374] integrating first working classification training model --- train/config_params.json | 20 ++- train/models.py | 69 +++++++- train/requirements.txt | 1 + train/train.py | 374 ++++++++++++++++++++++++--------------- train/utils.py | 113 ++++++++++++ 5 files changed, 419 insertions(+), 158 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index bd47a52..43ad1bc 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,13 +1,15 @@ { - "model_name" : "hybrid_transformer_cnn", + "model_name" : "resnet50_unet", + "task": "classification", "n_classes" : 2, - "n_epochs" : 2, - "input_height" : 448, - "input_width" : 448, + "n_epochs" : 7, + "input_height" : 224, + "input_width" : 224, "weight_decay" : 1e-6, - "n_batch" : 2, + "n_batch" : 6, "learning_rate": 1e-4, - "patches" : true, + "f1_threshold_classification": 0.8, + "patches" : false, "pretraining" : true, "augmentation" : false, "flip_aug" : false, @@ -33,7 +35,7 @@ "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "/train", - "dir_eval": "/eval", - "dir_output": "/out" + "dir_train": "/home/vahid/Downloads/image_classification_data/train", + "dir_eval": "/home/vahid/Downloads/image_classification_data/eval", + "dir_output": "/home/vahid/Downloads/image_classification_data/output" } diff --git a/train/models.py b/train/models.py index f7a7ad8..a6de1ef 100644 --- a/train/models.py +++ b/train/models.py @@ -400,7 +400,7 @@ def vit_resnet50_unet(n_classes,patch_size, num_patches, input_height=224,input_ f5 = x if pretraining: - model = keras.Model(inputs, x).load_weights(resnet50_Weights_path) + model = Model(inputs, x).load_weights(resnet50_Weights_path) num_patches = x.shape[1]*x.shape[2] patches = Patches(patch_size)(x) @@ -468,6 +468,71 @@ def vit_resnet50_unet(n_classes,patch_size, num_patches, input_height=224,input_ o = (BatchNormalization(axis=bn_axis))(o) o = (Activation('softmax'))(o) - model = keras.Model(inputs=inputs, outputs=o) + model = Model(inputs=inputs, outputs=o) + return model + +def resnet50_classifier(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + include_top=True + assert input_height%32 == 0 + assert input_width%32 == 0 + + + img_input = Input(shape=(input_height,input_width , 3 )) + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x) + + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x ) + + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + if pretraining: + Model(img_input, x).load_weights(resnet50_Weights_path) + + x = AveragePooling2D((7, 7), name='avg_pool')(x) + x = Flatten()(x) + + ## + x = Dense(256, activation='relu', name='fc512')(x) + x=Dropout(0.2)(x) + ## + x = Dense(n_classes, activation='softmax', name='fc1000')(x) + model = Model(img_input, x) + + + + return model diff --git a/train/requirements.txt b/train/requirements.txt index 20b6a32..3e56438 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -6,3 +6,4 @@ tqdm imutils numpy scipy +scikit-learn diff --git a/train/train.py b/train/train.py index 6e6a172..efcd3ac 100644 --- a/train/train.py +++ b/train/train.py @@ -11,6 +11,7 @@ from metrics import * from tensorflow.keras.models import load_model from tqdm import tqdm import json +from sklearn.metrics import f1_score def configuration(): @@ -73,6 +74,8 @@ def config_params(): is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. weighted_loss = False # Use weighted categorical cross entropy as loss fucntion. When set to true, "is_loss_soft_dice" must be false. data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". + task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification. + f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. @ex.automain @@ -86,162 +89,239 @@ def run(_config, n_classes, n_epochs, input_height, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, continue_training, transformer_patchsize, num_patches_xy, model_name, flip_index, dir_eval, dir_output, - pretraining, learning_rate): + pretraining, learning_rate, task, f1_threshold_classification): - num_patches = num_patches_xy[0]*num_patches_xy[1] - if data_is_provided: - dir_train_flowing = os.path.join(dir_output, 'train') - dir_eval_flowing = os.path.join(dir_output, 'eval') - - dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') - dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') - - dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images') - dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels') - - configuration() - - else: - dir_img, dir_seg = get_dirs_or_files(dir_train) - dir_img_val, dir_seg_val = get_dirs_or_files(dir_eval) - - # make first a directory in output for both training and evaluations in order to flow data from these directories. - dir_train_flowing = os.path.join(dir_output, 'train') - dir_eval_flowing = os.path.join(dir_output, 'eval') - - dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images/') - dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels/') - - dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images/') - dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels/') - - if os.path.isdir(dir_train_flowing): - os.system('rm -rf ' + dir_train_flowing) - os.makedirs(dir_train_flowing) - else: - os.makedirs(dir_train_flowing) - - if os.path.isdir(dir_eval_flowing): - os.system('rm -rf ' + dir_eval_flowing) - os.makedirs(dir_eval_flowing) - else: - os.makedirs(dir_eval_flowing) - - os.mkdir(dir_flow_train_imgs) - os.mkdir(dir_flow_train_labels) - - os.mkdir(dir_flow_eval_imgs) - os.mkdir(dir_flow_eval_labels) - - # set the gpu configuration - configuration() + if task == "segmentation": - imgs_list=np.array(os.listdir(dir_img)) - segs_list=np.array(os.listdir(dir_seg)) - - imgs_list_test=np.array(os.listdir(dir_img_val)) - segs_list_test=np.array(os.listdir(dir_seg_val)) - - # writing patches into a sub-folder in order to be flowed from directory. - provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, - dir_flow_train_labels, input_height, input_width, blur_k, - blur_aug, padding_white, padding_black, flip_aug, binarization, - scaling, degrading, brightening, scales, degrade_scales, brightness, - flip_index, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, augmentation=augmentation, - patches=patches) - - provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, - dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, - blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, - scaling, degrading, brightening, scales, degrade_scales, brightness, - flip_index, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, augmentation=False, patches=patches) - - if weighted_loss: - weights = np.zeros(n_classes) + num_patches = num_patches_xy[0]*num_patches_xy[1] if data_is_provided: - for obj in os.listdir(dir_flow_train_labels): - try: - label_obj = cv2.imread(dir_flow_train_labels + '/' + obj) - label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) - weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) - except: - pass + dir_train_flowing = os.path.join(dir_output, 'train') + dir_eval_flowing = os.path.join(dir_output, 'eval') + + dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') + dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') + + dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images') + dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels') + + configuration() + else: + dir_img, dir_seg = get_dirs_or_files(dir_train) + dir_img_val, dir_seg_val = get_dirs_or_files(dir_eval) - for obj in os.listdir(dir_seg): - try: - label_obj = cv2.imread(dir_seg + '/' + obj) - label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) - weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) - except: - pass + # make first a directory in output for both training and evaluations in order to flow data from these directories. + dir_train_flowing = os.path.join(dir_output, 'train') + dir_eval_flowing = os.path.join(dir_output, 'eval') - weights = 1.00 / weights + dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images/') + dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels/') - weights = weights / float(np.sum(weights)) - weights = weights / float(np.min(weights)) - weights = weights / float(np.sum(weights)) + dir_flow_eval_imgs = os.path.join(dir_eval_flowing, 'images/') + dir_flow_eval_labels = os.path.join(dir_eval_flowing, 'labels/') - if continue_training: - if model_name=='resnet50_unet': - if is_loss_soft_dice: - model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) - if weighted_loss: - model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - if not is_loss_soft_dice and not weighted_loss: - model = load_model(dir_of_start_model , compile=True) - elif model_name=='hybrid_transformer_cnn': - if is_loss_soft_dice: - model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) - if weighted_loss: - model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) - if not is_loss_soft_dice and not weighted_loss: - model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) - else: - index_start = 0 - if model_name=='resnet50_unet': - model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) - elif model_name=='hybrid_transformer_cnn': - model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width,weight_decay,pretraining) - - #if you want to see the model structure just uncomment model summary. - #model.summary() - + if os.path.isdir(dir_train_flowing): + os.system('rm -rf ' + dir_train_flowing) + os.makedirs(dir_train_flowing) + else: + os.makedirs(dir_train_flowing) - if not is_loss_soft_dice and not weighted_loss: + if os.path.isdir(dir_eval_flowing): + os.system('rm -rf ' + dir_eval_flowing) + os.makedirs(dir_eval_flowing) + else: + os.makedirs(dir_eval_flowing) + + os.mkdir(dir_flow_train_imgs) + os.mkdir(dir_flow_train_labels) + + os.mkdir(dir_flow_eval_imgs) + os.mkdir(dir_flow_eval_labels) + + # set the gpu configuration + configuration() + + imgs_list=np.array(os.listdir(dir_img)) + segs_list=np.array(os.listdir(dir_seg)) + + imgs_list_test=np.array(os.listdir(dir_img_val)) + segs_list_test=np.array(os.listdir(dir_seg_val)) + + # writing patches into a sub-folder in order to be flowed from directory. + provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, + dir_flow_train_labels, input_height, input_width, blur_k, + blur_aug, padding_white, padding_black, flip_aug, binarization, + scaling, degrading, brightening, scales, degrade_scales, brightness, + flip_index, scaling_bluring, scaling_brightness, scaling_binarization, + rotation, rotation_not_90, thetha, scaling_flip, augmentation=augmentation, + patches=patches) + + provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, + dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, + blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, + scaling, degrading, brightening, scales, degrade_scales, brightness, + flip_index, scaling_bluring, scaling_brightness, scaling_binarization, + rotation, rotation_not_90, thetha, scaling_flip, augmentation=False, patches=patches) + + if weighted_loss: + weights = np.zeros(n_classes) + if data_is_provided: + for obj in os.listdir(dir_flow_train_labels): + try: + label_obj = cv2.imread(dir_flow_train_labels + '/' + obj) + label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) + weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) + except: + pass + else: + + for obj in os.listdir(dir_seg): + try: + label_obj = cv2.imread(dir_seg + '/' + obj) + label_obj_one_hot = get_one_hot(label_obj, label_obj.shape[0], label_obj.shape[1], n_classes) + weights += (label_obj_one_hot.sum(axis=0)).sum(axis=0) + except: + pass + + weights = 1.00 / weights + + weights = weights / float(np.sum(weights)) + weights = weights / float(np.min(weights)) + weights = weights / float(np.sum(weights)) + + if continue_training: + if model_name=='resnet50_unet': + if is_loss_soft_dice: + model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) + if weighted_loss: + model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + if not is_loss_soft_dice and not weighted_loss: + model = load_model(dir_of_start_model , compile=True) + elif model_name=='hybrid_transformer_cnn': + if is_loss_soft_dice: + model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) + if weighted_loss: + model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) + if not is_loss_soft_dice and not weighted_loss: + model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) + else: + index_start = 0 + if model_name=='resnet50_unet': + model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + elif model_name=='hybrid_transformer_cnn': + model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width,weight_decay,pretraining) + + #if you want to see the model structure just uncomment model summary. + #model.summary() + + + if not is_loss_soft_dice and not weighted_loss: + model.compile(loss='categorical_crossentropy', + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if is_loss_soft_dice: + model.compile(loss=soft_dice_loss, + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if weighted_loss: + model.compile(loss=weighted_categorical_crossentropy(weights), + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + + # generating train and evaluation data + train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, batch_size=n_batch, + input_height=input_height, input_width=input_width, n_classes=n_classes) + val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, batch_size=n_batch, + input_height=input_height, input_width=input_width, n_classes=n_classes) + + ##img_validation_patches = os.listdir(dir_flow_eval_imgs) + ##score_best=[] + ##score_best.append(0) + for i in tqdm(range(index_start, n_epochs + index_start)): + model.fit_generator( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, + validation_data=val_gen, + validation_steps=1, + epochs=1) + model.save(dir_output+'/'+'model_'+str(i)) + + with open(dir_output+'/'+'model_'+str(i)+'/'+"config.json", "w") as fp: + json.dump(_config, fp) # encode dict into JSON + + #os.system('rm -rf '+dir_train_flowing) + #os.system('rm -rf '+dir_eval_flowing) + + #model.save(dir_output+'/'+'model'+'.h5') + elif task=='classification': + configuration() + model = resnet50_classifier(n_classes, input_height, input_width,weight_decay,pretraining) + + opt_adam = Adam(learning_rate=0.001) model.compile(loss='categorical_crossentropy', - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - if is_loss_soft_dice: - model.compile(loss=soft_dice_loss, - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - if weighted_loss: - model.compile(loss=weighted_categorical_crossentropy(weights), - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - - # generating train and evaluation data - train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, batch_size=n_batch, - input_height=input_height, input_width=input_width, n_classes=n_classes) - val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, batch_size=n_batch, - input_height=input_height, input_width=input_width, n_classes=n_classes) - - ##img_validation_patches = os.listdir(dir_flow_eval_imgs) - ##score_best=[] - ##score_best.append(0) - for i in tqdm(range(index_start, n_epochs + index_start)): - model.fit_generator( - train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, - validation_data=val_gen, - validation_steps=1, - epochs=1) - model.save(dir_output+'/'+'model_'+str(i)) - - with open(dir_output+'/'+'model_'+str(i)+'/'+"config.json", "w") as fp: - json.dump(_config, fp) # encode dict into JSON + optimizer = opt_adam,metrics=['accuracy']) - #os.system('rm -rf '+dir_train_flowing) - #os.system('rm -rf '+dir_eval_flowing) - #model.save(dir_output+'/'+'model'+'.h5') + testX, testY = generate_data_from_folder_evaluation(dir_eval, input_height, input_width, n_classes) + + #print(testY.shape, testY) + + y_tot=np.zeros((testX.shape[0],n_classes)) + indexer=0 + + score_best=[] + score_best.append(0) + + num_rows = return_number_of_total_training_data(dir_train) + + weights=[] + + for i in range(n_epochs): + #history = model.fit(trainX, trainY, epochs=1, batch_size=n_batch, validation_data=(testX, testY), verbose=2)#,class_weight=weights) + history = model.fit( generate_data_from_folder_training(dir_train, n_batch , input_height, input_width, n_classes), steps_per_epoch=num_rows / n_batch, verbose=0)#,class_weight=weights) + + y_pr_class = [] + for jj in range(testY.shape[0]): + y_pr=model.predict(testX[jj,:,:,:].reshape(1,input_height,input_width,3), verbose=0) + y_pr_ind= np.argmax(y_pr,axis=1) + #print(y_pr_ind, 'y_pr_ind') + y_pr_class.append(y_pr_ind) + + + y_pr_class = np.array(y_pr_class) + #model.save('./models_save/model_'+str(i)+'.h5') + #y_pr_class=np.argmax(y_pr,axis=1) + f1score=f1_score(np.argmax(testY,axis=1), y_pr_class, average='macro') + + print(i,f1score) + + if f1score>score_best[0]: + score_best[0]=f1score + model.save(os.path.join(dir_output,'model_best')) + + + ##best_model=keras.models.clone_model(model) + ##best_model.build() + ##best_model.set_weights(model.get_weights()) + if f1score > f1_threshold_classification: + weights.append(model.get_weights() ) + y_tot=y_tot+y_pr + + indexer+=1 + y_tot=y_tot/float(indexer) + + + new_weights=list() + + for weights_list_tuple in zip(*weights): + new_weights.append( [np.array(weights_).mean(axis=0) for weights_ in zip(*weights_list_tuple)] ) + + new_weights = [np.array(x) for x in new_weights] + + model_weight_averaged=tf.keras.models.clone_model(model) + + model_weight_averaged.set_weights(new_weights) + + #y_tot_end=np.argmax(y_tot,axis=1) + #print(f1_score(np.argmax(testY,axis=1), y_tot_end, average='macro')) + + ##best_model.save('model_taza.h5') + model_weight_averaged.save(os.path.join(dir_output,'model_ens_avg')) + diff --git a/train/utils.py b/train/utils.py index c2786ec..af3c5f8 100644 --- a/train/utils.py +++ b/train/utils.py @@ -8,6 +8,119 @@ import random from tqdm import tqdm import imutils import math +from tensorflow.keras.utils import to_categorical + + +def return_number_of_total_training_data(path_classes): + sub_classes = os.listdir(path_classes) + n_tot = 0 + for sub_c in sub_classes: + sub_files = os.listdir(os.path.join(path_classes,sub_c)) + n_tot = n_tot + len(sub_files) + return n_tot + + + +def generate_data_from_folder_evaluation(path_classes, height, width, n_classes): + sub_classes = os.listdir(path_classes) + #n_classes = len(sub_classes) + all_imgs = [] + labels = [] + dicts =dict() + indexer= 0 + for sub_c in sub_classes: + sub_files = os.listdir(os.path.join(path_classes,sub_c )) + sub_files = [os.path.join(path_classes,sub_c )+'/' + x for x in sub_files] + #print( os.listdir(os.path.join(path_classes,sub_c )) ) + all_imgs = all_imgs + sub_files + sub_labels = list( np.zeros( len(sub_files) ) +indexer ) + + #print( len(sub_labels) ) + labels = labels + sub_labels + dicts[sub_c] = indexer + indexer +=1 + + + categories = to_categorical(range(n_classes)).astype(np.int16)#[ [1 , 0, 0 , 0 , 0 , 0] , [0 , 1, 0 , 0 , 0 , 0] , [0 , 0, 1 , 0 , 0 , 0] , [0 , 0, 0 , 1 , 0 , 0] , [0 , 0, 0 , 0 , 1 , 0] , [0 , 0, 0 , 0 , 0 , 1] ] + ret_x= np.zeros((len(labels), height,width, 3)).astype(np.int16) + ret_y= np.zeros((len(labels), n_classes)).astype(np.int16) + + #print(all_imgs) + for i in range(len(all_imgs)): + row = all_imgs[i] + #####img = cv2.imread(row, 0) + #####img= resize_image (img, height, width) + #####img = img.astype(np.uint16) + #####ret_x[i, :,:,0] = img[:,:] + #####ret_x[i, :,:,1] = img[:,:] + #####ret_x[i, :,:,2] = img[:,:] + + img = cv2.imread(row) + img= resize_image (img, height, width) + img = img.astype(np.uint16) + ret_x[i, :,:] = img[:,:,:] + + ret_y[i, :] = categories[ int( labels[i] ) ][:] + + return ret_x/255., ret_y + +def generate_data_from_folder_training(path_classes, batchsize, height, width, n_classes): + sub_classes = os.listdir(path_classes) + n_classes = len(sub_classes) + + all_imgs = [] + labels = [] + dicts =dict() + indexer= 0 + for sub_c in sub_classes: + sub_files = os.listdir(os.path.join(path_classes,sub_c )) + sub_files = [os.path.join(path_classes,sub_c )+'/' + x for x in sub_files] + #print( os.listdir(os.path.join(path_classes,sub_c )) ) + all_imgs = all_imgs + sub_files + sub_labels = list( np.zeros( len(sub_files) ) +indexer ) + + #print( len(sub_labels) ) + labels = labels + sub_labels + dicts[sub_c] = indexer + indexer +=1 + + ids = np.array(range(len(labels))) + random.shuffle(ids) + + shuffled_labels = np.array(labels)[ids] + shuffled_files = np.array(all_imgs)[ids] + categories = to_categorical(range(n_classes)).astype(np.int16)#[ [1 , 0, 0 , 0 , 0 , 0] , [0 , 1, 0 , 0 , 0 , 0] , [0 , 0, 1 , 0 , 0 , 0] , [0 , 0, 0 , 1 , 0 , 0] , [0 , 0, 0 , 0 , 1 , 0] , [0 , 0, 0 , 0 , 0 , 1] ] + ret_x= np.zeros((batchsize, height,width, 3)).astype(np.int16) + ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + batchcount = 0 + while True: + for i in range(len(shuffled_files)): + row = shuffled_files[i] + #print(row) + ###img = cv2.imread(row, 0) + ###img= resize_image (img, height, width) + ###img = img.astype(np.uint16) + ###ret_x[batchcount, :,:,0] = img[:,:] + ###ret_x[batchcount, :,:,1] = img[:,:] + ###ret_x[batchcount, :,:,2] = img[:,:] + + img = cv2.imread(row) + img= resize_image (img, height, width) + img = img.astype(np.uint16) + ret_x[batchcount, :,:,:] = img[:,:,:] + + #print(int(shuffled_labels[i]) ) + #print( categories[int(shuffled_labels[i])] ) + ret_y[batchcount, :] = categories[ int( shuffled_labels[i] ) ][:] + + batchcount+=1 + + if batchcount>=batchsize: + ret_x = ret_x/255. + yield (ret_x, ret_y) + ret_x= np.zeros((batchsize, height,width, 3)).astype(np.int16) + ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + batchcount = 0 def do_brightening(img_in_dir, factor): im = Image.open(img_in_dir) From c989f7ac6111314a394700e833abe351f5daae43 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 6 May 2024 18:31:48 +0200 Subject: [PATCH 041/374] adding enhancement training --- train/config_params.json | 20 +++++----- train/gt_for_enhancement_creator.py | 31 +++++++++++++++ train/models.py | 27 ++++++++----- train/train.py | 47 ++++++++++++---------- train/utils.py | 62 ++++++++++++++++------------- 5 files changed, 119 insertions(+), 68 deletions(-) create mode 100644 train/gt_for_enhancement_creator.py diff --git a/train/config_params.json b/train/config_params.json index 43ad1bc..1c7a940 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,15 +1,15 @@ { "model_name" : "resnet50_unet", - "task": "classification", - "n_classes" : 2, - "n_epochs" : 7, - "input_height" : 224, - "input_width" : 224, + "task": "enhancement", + "n_classes" : 3, + "n_epochs" : 3, + "input_height" : 448, + "input_width" : 448, "weight_decay" : 1e-6, - "n_batch" : 6, + "n_batch" : 3, "learning_rate": 1e-4, "f1_threshold_classification": 0.8, - "patches" : false, + "patches" : true, "pretraining" : true, "augmentation" : false, "flip_aug" : false, @@ -35,7 +35,7 @@ "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "/home/vahid/Downloads/image_classification_data/train", - "dir_eval": "/home/vahid/Downloads/image_classification_data/eval", - "dir_output": "/home/vahid/Downloads/image_classification_data/output" + "dir_train": "./training_data_sample_enhancement", + "dir_eval": "./eval", + "dir_output": "./out" } diff --git a/train/gt_for_enhancement_creator.py b/train/gt_for_enhancement_creator.py new file mode 100644 index 0000000..9a4274f --- /dev/null +++ b/train/gt_for_enhancement_creator.py @@ -0,0 +1,31 @@ +import cv2 +import os + +def resize_image(seg_in, input_height, input_width): + return cv2.resize(seg_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) + + +dir_imgs = './training_data_sample_enhancement/images' +dir_out_imgs = './training_data_sample_enhancement/images_gt' +dir_out_labs = './training_data_sample_enhancement/labels_gt' + +ls_imgs = os.listdir(dir_imgs) + + +ls_scales = [ 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] + + +for img in ls_imgs: + img_name = img.split('.')[0] + img_type = img.split('.')[1] + image = cv2.imread(os.path.join(dir_imgs, img)) + for i, scale in enumerate(ls_scales): + height_sc = int(image.shape[0]*scale) + width_sc = int(image.shape[1]*scale) + + image_down_scaled = resize_image(image, height_sc, width_sc) + image_back_to_org_scale = resize_image(image_down_scaled, image.shape[0], image.shape[1]) + + cv2.imwrite(os.path.join(dir_out_imgs, img_name+'_'+str(i)+'.'+img_type), image_back_to_org_scale) + cv2.imwrite(os.path.join(dir_out_labs, img_name+'_'+str(i)+'.'+img_type), image) + diff --git a/train/models.py b/train/models.py index a6de1ef..4cceacd 100644 --- a/train/models.py +++ b/train/models.py @@ -168,7 +168,7 @@ def conv_block(input_tensor, kernel_size, filters, stage, block, strides=(2, 2)) return x -def resnet50_unet_light(n_classes, input_height=224, input_width=224, weight_decay=1e-6, pretraining=False): +def resnet50_unet_light(n_classes, input_height=224, input_width=224, taks="segmentation", weight_decay=1e-6, pretraining=False): assert input_height % 32 == 0 assert input_width % 32 == 0 @@ -259,14 +259,17 @@ def resnet50_unet_light(n_classes, input_height=224, input_width=224, weight_dec o = Activation('relu')(o) o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(o) - o = (BatchNormalization(axis=bn_axis))(o) - o = (Activation('softmax'))(o) + if task == "segmentation": + o = (BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + else: + o = (Activation('sigmoid'))(o) model = Model(img_input, o) return model -def resnet50_unet(n_classes, input_height=224, input_width=224, weight_decay=1e-6, pretraining=False): +def resnet50_unet(n_classes, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): assert input_height % 32 == 0 assert input_width % 32 == 0 @@ -354,15 +357,18 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, weight_decay=1e- o = Activation('relu')(o) o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay))(o) - o = (BatchNormalization(axis=bn_axis))(o) - o = (Activation('softmax'))(o) + if task == "segmentation": + o = (BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + else: + o = (Activation('sigmoid'))(o) model = Model(img_input, o) return model -def vit_resnet50_unet(n_classes,patch_size, num_patches, input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): +def vit_resnet50_unet(n_classes, patch_size, num_patches, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): inputs = layers.Input(shape=(input_height, input_width, 3)) IMAGE_ORDERING = 'channels_last' bn_axis=3 @@ -465,8 +471,11 @@ def vit_resnet50_unet(n_classes,patch_size, num_patches, input_height=224,input_ o = Activation('relu')(o) o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(o) - o = (BatchNormalization(axis=bn_axis))(o) - o = (Activation('softmax'))(o) + if task == "segmentation": + o = (BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + else: + o = (Activation('sigmoid'))(o) model = Model(inputs=inputs, outputs=o) diff --git a/train/train.py b/train/train.py index efcd3ac..595debe 100644 --- a/train/train.py +++ b/train/train.py @@ -1,5 +1,6 @@ import os import sys +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import tensorflow as tf from tensorflow.compat.v1.keras.backend import set_session import warnings @@ -91,7 +92,7 @@ def run(_config, n_classes, n_epochs, input_height, num_patches_xy, model_name, flip_index, dir_eval, dir_output, pretraining, learning_rate, task, f1_threshold_classification): - if task == "segmentation": + if task == "segmentation" or "enhancement": num_patches = num_patches_xy[0]*num_patches_xy[1] if data_is_provided: @@ -153,7 +154,7 @@ def run(_config, n_classes, n_epochs, input_height, blur_aug, padding_white, padding_black, flip_aug, binarization, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, augmentation=augmentation, + rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=augmentation, patches=patches) provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, @@ -161,7 +162,7 @@ def run(_config, n_classes, n_epochs, input_height, blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, augmentation=False, patches=patches) + rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches) if weighted_loss: weights = np.zeros(n_classes) @@ -191,45 +192,49 @@ def run(_config, n_classes, n_epochs, input_height, if continue_training: if model_name=='resnet50_unet': - if is_loss_soft_dice: + if is_loss_soft_dice and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) - if weighted_loss: + if weighted_loss and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) if not is_loss_soft_dice and not weighted_loss: model = load_model(dir_of_start_model , compile=True) elif model_name=='hybrid_transformer_cnn': - if is_loss_soft_dice: + if is_loss_soft_dice and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) - if weighted_loss: + if weighted_loss and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) if not is_loss_soft_dice and not weighted_loss: model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) else: index_start = 0 if model_name=='resnet50_unet': - model = resnet50_unet(n_classes, input_height, input_width,weight_decay,pretraining) + model = resnet50_unet(n_classes, input_height, input_width, task, weight_decay, pretraining) elif model_name=='hybrid_transformer_cnn': - model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width,weight_decay,pretraining) + model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width, task, weight_decay, pretraining) #if you want to see the model structure just uncomment model summary. #model.summary() - - if not is_loss_soft_dice and not weighted_loss: - model.compile(loss='categorical_crossentropy', - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - if is_loss_soft_dice: - model.compile(loss=soft_dice_loss, - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) - if weighted_loss: - model.compile(loss=weighted_categorical_crossentropy(weights), - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if task == "segmentation": + if not is_loss_soft_dice and not weighted_loss: + model.compile(loss='categorical_crossentropy', + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if is_loss_soft_dice: + model.compile(loss=soft_dice_loss, + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + if weighted_loss: + model.compile(loss=weighted_categorical_crossentropy(weights), + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + elif task == "enhancement": + model.compile(loss='mean_squared_error', + optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + # generating train and evaluation data train_gen = data_gen(dir_flow_train_imgs, dir_flow_train_labels, batch_size=n_batch, - input_height=input_height, input_width=input_width, n_classes=n_classes) + input_height=input_height, input_width=input_width, n_classes=n_classes, task=task) val_gen = data_gen(dir_flow_eval_imgs, dir_flow_eval_labels, batch_size=n_batch, - input_height=input_height, input_width=input_width, n_classes=n_classes) + input_height=input_height, input_width=input_width, n_classes=n_classes, task=task) ##img_validation_patches = os.listdir(dir_flow_eval_imgs) ##score_best=[] diff --git a/train/utils.py b/train/utils.py index af3c5f8..0c5a458 100644 --- a/train/utils.py +++ b/train/utils.py @@ -268,7 +268,7 @@ def IoU(Yi, y_predi): return mIoU -def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_classes): +def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_classes, task='segmentation'): c = 0 n = [f for f in os.listdir(img_folder) if not f.startswith('.')] # os.listdir(img_folder) #List of training images random.shuffle(n) @@ -277,8 +277,6 @@ def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_c mask = np.zeros((batch_size, input_height, input_width, n_classes)).astype('float') for i in range(c, c + batch_size): # initially from 0 to 16, c = 0. - # print(img_folder+'/'+n[i]) - try: filename = n[i].split('.')[0] @@ -287,11 +285,14 @@ def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_c interpolation=cv2.INTER_NEAREST) # Read an image from folder and resize img[i - c] = train_img # add to array - img[0], img[1], and so on. - train_mask = cv2.imread(mask_folder + '/' + filename + '.png') - # print(mask_folder+'/'+filename+'.png') - # print(train_mask.shape) - train_mask = get_one_hot(resize_image(train_mask, input_height, input_width), input_height, input_width, - n_classes) + if task == "segmentation": + train_mask = cv2.imread(mask_folder + '/' + filename + '.png') + train_mask = get_one_hot(resize_image(train_mask, input_height, input_width), input_height, input_width, + n_classes) + elif task == "enhancement": + train_mask = cv2.imread(mask_folder + '/' + filename + '.png')/255. + train_mask = resize_image(train_mask, input_height, input_width) + # train_mask = train_mask.reshape(224, 224, 1) # Add extra dimension for parity with train_img size [512 * 512 * 3] mask[i - c] = train_mask @@ -539,14 +540,19 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow padding_white, padding_black, flip_aug, binarization, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, scaling_bluring, scaling_brightness, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, augmentation=False, patches=False): + rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False): indexer = 0 for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): img_name = im.split('.')[0] + if task == "segmentation": + dir_of_label_file = os.path.join(dir_seg, img_name + '.png') + elif task=="enhancement": + dir_of_label_file = os.path.join(dir_seg, im) + if not patches: cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_img + '/' + im), input_height, input_width)) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) indexer += 1 if augmentation: @@ -556,7 +562,7 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow resize_image(cv2.flip(cv2.imread(dir_img+'/'+im),f_i),input_height,input_width) ) cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), input_height, input_width)) + resize_image(cv2.flip(cv2.imread(dir_of_label_file), f_i), input_height, input_width)) indexer += 1 if blur_aug: @@ -565,7 +571,7 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow (resize_image(bluring(cv2.imread(dir_img + '/' + im), blur_i), input_height, input_width))) cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) indexer += 1 if binarization: @@ -573,26 +579,26 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow resize_image(otsu_copy(cv2.imread(dir_img + '/' + im)), input_height, input_width)) cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_seg + '/' + img_name + '.png'), input_height, input_width)) + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) indexer += 1 if patches: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - cv2.imread(dir_img + '/' + im), cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_img + '/' + im), cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer) if augmentation: if rotation: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, rotation_90(cv2.imread(dir_img + '/' + im)), - rotation_90(cv2.imread(dir_seg + '/' + img_name + '.png')), + rotation_90(cv2.imread(dir_of_label_file)), input_height, input_width, indexer=indexer) if rotation_not_90: for thetha_i in thetha: img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/'+im), - cv2.imread(dir_seg + '/'+img_name + '.png'), thetha_i) + cv2.imread(dir_of_label_file), thetha_i) indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, img_max_rotated, label_max_rotated, @@ -601,24 +607,24 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow for f_i in flip_index: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, cv2.flip(cv2.imread(dir_img + '/' + im), f_i), - cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), + cv2.flip(cv2.imread(dir_of_label_file), f_i), input_height, input_width, indexer=indexer) if blur_aug: for blur_i in blur_k: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, bluring(cv2.imread(dir_img + '/' + im), blur_i), - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer) if padding_black: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, do_padding_black(cv2.imread(dir_img + '/' + im)), - do_padding_label(cv2.imread(dir_seg + '/' + img_name + '.png')), + do_padding_label(cv2.imread(dir_of_label_file)), input_height, input_width, indexer=indexer) if padding_white: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, do_padding_white(cv2.imread(dir_img + '/'+im)), - do_padding_label(cv2.imread(dir_seg + '/' + img_name + '.png')), + do_padding_label(cv2.imread(dir_of_label_file)), input_height, input_width, indexer=indexer) if brightening: @@ -626,7 +632,7 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow try: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, do_brightening(dir_img + '/' +im, factor), - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer) except: pass @@ -634,20 +640,20 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow for sc_ind in scales: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, cv2.imread(dir_img + '/' + im) , - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer, scaler=sc_ind) if degrading: for degrade_scale_ind in degrade_scales: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, do_degrading(cv2.imread(dir_img + '/' + im), degrade_scale_ind), - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer) if binarization: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, otsu_copy(cv2.imread(dir_img + '/' + im)), - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer) if scaling_brightness: @@ -657,7 +663,7 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, do_brightening(dir_img + '/' + im, factor) - ,cv2.imread(dir_seg + '/' + img_name + '.png') + ,cv2.imread(dir_of_label_file) ,input_height, input_width, indexer=indexer, scaler=sc_ind) except: pass @@ -667,14 +673,14 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow for blur_i in blur_k: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, bluring(cv2.imread(dir_img + '/' + im), blur_i), - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer, scaler=sc_ind) if scaling_binarization: for sc_ind in scales: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, otsu_copy(cv2.imread(dir_img + '/' + im)), - cv2.imread(dir_seg + '/' + img_name + '.png'), + cv2.imread(dir_of_label_file), input_height, input_width, indexer=indexer, scaler=sc_ind) if scaling_flip: @@ -682,5 +688,5 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow for f_i in flip_index: indexer = get_patches_num_scale_new(dir_flow_train_imgs, dir_flow_train_labels, cv2.flip( cv2.imread(dir_img + '/' + im), f_i), - cv2.flip(cv2.imread(dir_seg + '/' + img_name + '.png'), f_i), + cv2.flip(cv2.imread(dir_of_label_file), f_i), input_height, input_width, indexer=indexer, scaler=sc_ind) From e1f62c2e9827030e3386ff678a131481d70e8e14 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 7 May 2024 13:34:03 +0200 Subject: [PATCH 042/374] inference script is added --- train/config_params.json | 17 +- train/inference.py | 490 +++++++++++++++++++++++++++++++++++++++ train/train.py | 42 ++-- train/utils.py | 30 +-- 4 files changed, 537 insertions(+), 42 deletions(-) create mode 100644 train/inference.py diff --git a/train/config_params.json b/train/config_params.json index 1c7a940..8a56de5 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,12 +1,12 @@ { - "model_name" : "resnet50_unet", - "task": "enhancement", - "n_classes" : 3, - "n_epochs" : 3, + "backbone_type" : "nontransformer", + "task": "classification", + "n_classes" : 2, + "n_epochs" : 20, "input_height" : 448, "input_width" : 448, "weight_decay" : 1e-6, - "n_batch" : 3, + "n_batch" : 6, "learning_rate": 1e-4, "f1_threshold_classification": 0.8, "patches" : true, @@ -21,7 +21,7 @@ "scaling_flip" : false, "rotation": false, "rotation_not_90": false, - "num_patches_xy": [28, 28], + "transformer_num_patches_xy": [28, 28], "transformer_patchsize": 1, "blur_k" : ["blur","guass","median"], "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], @@ -29,13 +29,14 @@ "degrade_scales" : [0.2, 0.4], "flip_index" : [0, 1, -1], "thetha" : [10, -10], + "classification_classes_name" : {"0":"apple", "1":"orange"}, "continue_training": false, "index_start" : 0, "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "./training_data_sample_enhancement", + "dir_train": "./train", "dir_eval": "./eval", - "dir_output": "./out" + "dir_output": "./output" } diff --git a/train/inference.py b/train/inference.py new file mode 100644 index 0000000..6911bea --- /dev/null +++ b/train/inference.py @@ -0,0 +1,490 @@ +#! /usr/bin/env python3 + +__version__= '1.0' + +import argparse +import sys +import os +import numpy as np +import warnings +import xml.etree.ElementTree as et +import pandas as pd +from tqdm import tqdm +import csv +import cv2 +import seaborn as sns +import matplotlib.pyplot as plt +from tensorflow.keras.models import load_model +import tensorflow as tf +from tensorflow.keras import backend as K +from tensorflow.keras import layers +import tensorflow.keras.losses +from tensorflow.keras.layers import * +import click +import json +from tensorflow.python.keras import backend as tensorflow_backend + + + + + + +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + +__doc__=\ +""" +Tool to load model and predict for given image. +""" + +projection_dim = 64 +patch_size = 1 +num_patches =28*28 +class Patches(layers.Layer): + def __init__(self, **kwargs): + super(Patches, self).__init__() + self.patch_size = patch_size + + def call(self, images): + print(tf.shape(images)[1],'images') + print(self.patch_size,'self.patch_size') + batch_size = tf.shape(images)[0] + patches = tf.image.extract_patches( + images=images, + sizes=[1, self.patch_size, self.patch_size, 1], + strides=[1, self.patch_size, self.patch_size, 1], + rates=[1, 1, 1, 1], + padding="VALID", + ) + patch_dims = patches.shape[-1] + print(patches.shape,patch_dims,'patch_dims') + patches = tf.reshape(patches, [batch_size, -1, patch_dims]) + return patches + def get_config(self): + + config = super().get_config().copy() + config.update({ + 'patch_size': self.patch_size, + }) + return config + + +class PatchEncoder(layers.Layer): + def __init__(self, **kwargs): + super(PatchEncoder, self).__init__() + self.num_patches = num_patches + self.projection = layers.Dense(units=projection_dim) + self.position_embedding = layers.Embedding( + input_dim=num_patches, output_dim=projection_dim + ) + + def call(self, patch): + positions = tf.range(start=0, limit=self.num_patches, delta=1) + encoded = self.projection(patch) + self.position_embedding(positions) + return encoded + def get_config(self): + + config = super().get_config().copy() + config.update({ + 'num_patches': self.num_patches, + 'projection': self.projection, + 'position_embedding': self.position_embedding, + }) + return config + + +class sbb_predict: + def __init__(self,image, model, task, config_params_model, patches='false',save='false', ground_truth=None,weights_dir=None ): + self.image=image + self.patches=patches + self.save=save + self.model_dir=model + self.ground_truth=ground_truth + self.weights_dir=weights_dir + self.task=task + self.config_params_model=config_params_model + + def resize_image(self,img_in,input_height,input_width): + return cv2.resize( img_in, ( input_width,input_height) ,interpolation=cv2.INTER_NEAREST) + + + def color_images(self,seg): + ann_u=range(self.n_classes) + if len(np.shape(seg))==3: + seg=seg[:,:,0] + + seg_img=np.zeros((np.shape(seg)[0],np.shape(seg)[1],3)).astype(np.uint8) + colors=sns.color_palette("hls", self.n_classes) + + for c in ann_u: + c=int(c) + segl=(seg==c) + seg_img[:,:,0][seg==c]=c + seg_img[:,:,1][seg==c]=c + seg_img[:,:,2][seg==c]=c + return seg_img + + def otsu_copy_binary(self,img): + img_r=np.zeros((img.shape[0],img.shape[1],3)) + img1=img[:,:,0] + + #print(img.min()) + #print(img[:,:,0].min()) + #blur = cv2.GaussianBlur(img,(5,5)) + #ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) + retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + + + + img_r[:,:,0]=threshold1 + img_r[:,:,1]=threshold1 + img_r[:,:,2]=threshold1 + #img_r=img_r/float(np.max(img_r))*255 + return img_r + + def otsu_copy(self,img): + img_r=np.zeros((img.shape[0],img.shape[1],3)) + #img1=img[:,:,0] + + #print(img.min()) + #print(img[:,:,0].min()) + #blur = cv2.GaussianBlur(img,(5,5)) + #ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold1 = cv2.threshold(img[:,:,0], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold2 = cv2.threshold(img[:,:,1], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + _, threshold3 = cv2.threshold(img[:,:,2], 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) + + + + img_r[:,:,0]=threshold1 + img_r[:,:,1]=threshold2 + img_r[:,:,2]=threshold3 + ###img_r=img_r/float(np.max(img_r))*255 + return img_r + + def soft_dice_loss(self,y_true, y_pred, epsilon=1e-6): + + axes = tuple(range(1, len(y_pred.shape)-1)) + + numerator = 2. * K.sum(y_pred * y_true, axes) + + denominator = K.sum(K.square(y_pred) + K.square(y_true), axes) + return 1.00 - K.mean(numerator / (denominator + epsilon)) # average over classes and batch + + def weighted_categorical_crossentropy(self,weights=None): + + def loss(y_true, y_pred): + labels_floats = tf.cast(y_true, tf.float32) + per_pixel_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels_floats,logits=y_pred) + + if weights is not None: + weight_mask = tf.maximum(tf.reduce_max(tf.constant( + np.array(weights, dtype=np.float32)[None, None, None]) + * labels_floats, axis=-1), 1.0) + per_pixel_loss = per_pixel_loss * weight_mask[:, :, :, None] + return tf.reduce_mean(per_pixel_loss) + return self.loss + + + def IoU(self,Yi,y_predi): + ## mean Intersection over Union + ## Mean IoU = TP/(FN + TP + FP) + + IoUs = [] + Nclass = np.unique(Yi) + for c in Nclass: + TP = np.sum( (Yi == c)&(y_predi==c) ) + FP = np.sum( (Yi != c)&(y_predi==c) ) + FN = np.sum( (Yi == c)&(y_predi != c)) + IoU = TP/float(TP + FP + FN) + if self.n_classes>2: + print("class {:02.0f}: #TP={:6.0f}, #FP={:6.0f}, #FN={:5.0f}, IoU={:4.3f}".format(c,TP,FP,FN,IoU)) + IoUs.append(IoU) + if self.n_classes>2: + mIoU = np.mean(IoUs) + print("_________________") + print("Mean IoU: {:4.3f}".format(mIoU)) + return mIoU + elif self.n_classes==2: + mIoU = IoUs[1] + print("_________________") + print("IoU: {:4.3f}".format(mIoU)) + return mIoU + + def start_new_session_and_model(self): + + config = tf.compat.v1.ConfigProto() + config.gpu_options.allow_growth = True + + session = tf.compat.v1.Session(config=config) # tf.InteractiveSession() + tensorflow_backend.set_session(session) + #tensorflow.keras.layers.custom_layer = PatchEncoder + #tensorflow.keras.layers.custom_layer = Patches + self.model = load_model(self.model_dir , compile=False,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) + #config = tf.ConfigProto() + #config.gpu_options.allow_growth=True + + #self.session = tf.InteractiveSession() + #keras.losses.custom_loss = self.weighted_categorical_crossentropy + #self.model = load_model(self.model_dir , compile=False) + + + ##if self.weights_dir!=None: + ##self.model.load_weights(self.weights_dir) + + if self.task != 'classification': + self.img_height=self.model.layers[len(self.model.layers)-1].output_shape[1] + self.img_width=self.model.layers[len(self.model.layers)-1].output_shape[2] + self.n_classes=self.model.layers[len(self.model.layers)-1].output_shape[3] + + def visualize_model_output(self, prediction, img, task): + if task == "binarization": + prediction = prediction * -1 + prediction = prediction + 1 + added_image = prediction * 255 + else: + unique_classes = np.unique(prediction[:,:,0]) + rgb_colors = {'0' : [255, 255, 255], + '1' : [255, 0, 0], + '2' : [255, 125, 0], + '3' : [255, 0, 125], + '4' : [125, 125, 125], + '5' : [125, 125, 0], + '6' : [0, 125, 255], + '7' : [0, 125, 0], + '8' : [125, 125, 125], + '9' : [0, 125, 255], + '10' : [125, 0, 125], + '11' : [0, 255, 0], + '12' : [0, 0, 255], + '13' : [0, 255, 255], + '14' : [255, 125, 125], + '15' : [255, 0, 255]} + + output = np.zeros(prediction.shape) + + for unq_class in unique_classes: + rgb_class_unique = rgb_colors[str(int(unq_class))] + output[:,:,0][prediction[:,:,0]==unq_class] = rgb_class_unique[0] + output[:,:,1][prediction[:,:,0]==unq_class] = rgb_class_unique[1] + output[:,:,2][prediction[:,:,0]==unq_class] = rgb_class_unique[2] + + + + img = self.resize_image(img, output.shape[0], output.shape[1]) + + output = output.astype(np.int32) + img = img.astype(np.int32) + + + + added_image = cv2.addWeighted(img,0.5,output,0.1,0) + + return added_image + + def predict(self): + self.start_new_session_and_model() + if self.task == 'classification': + classes_names = self.config_params_model['classification_classes_name'] + img_1ch = img=cv2.imread(self.image, 0) + + img_1ch = img_1ch / 255.0 + img_1ch = cv2.resize(img_1ch, (self.config_params_model['input_height'], self.config_params_model['input_width']), interpolation=cv2.INTER_NEAREST) + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = self.model.predict(img_in, verbose=0) + index_class = np.argmax(label_p_pred[0]) + + print("Predicted Class: {}".format(classes_names[str(int(index_class))])) + else: + if self.patches: + #def textline_contours(img,input_width,input_height,n_classes,model): + + img=cv2.imread(self.image) + self.img_org = np.copy(img) + + if img.shape[0] < self.img_height: + img = cv2.resize(img, (img.shape[1], self.img_width), interpolation=cv2.INTER_NEAREST) + + if img.shape[1] < self.img_width: + img = cv2.resize(img, (self.img_height, img.shape[0]), interpolation=cv2.INTER_NEAREST) + margin = int(0 * self.img_width) + width_mid = self.img_width - 2 * margin + height_mid = self.img_height - 2 * margin + img = img / float(255.0) + + img_h = img.shape[0] + img_w = img.shape[1] + + prediction_true = np.zeros((img_h, img_w, 3)) + nxf = img_w / float(width_mid) + nyf = img_h / float(height_mid) + + nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) + nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + + for i in range(nxf): + for j in range(nyf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + self.img_width + else: + index_x_d = i * width_mid + index_x_u = index_x_d + self.img_width + if j == 0: + index_y_d = j * height_mid + index_y_u = index_y_d + self.img_height + else: + index_y_d = j * height_mid + index_y_u = index_y_d + self.img_height + + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - self.img_width + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - self.img_height + + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + label_p_pred = self.model.predict(img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2]), + verbose=0) + + if self.task == 'enhancement': + seg = label_p_pred[0, :, :, :] + seg = seg * 255 + elif self.task == 'segmentation' or self.task == 'binarization': + seg = np.argmax(label_p_pred, axis=3)[0] + seg = np.repeat(seg[:, :, np.newaxis], 3, axis=2) + + + if i == 0 and j == 0: + seg = seg[0 : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg + elif i == nxf - 1 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - 0] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0, :] = seg + elif i == 0 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin, :] = seg + elif i == nxf - 1 and j == 0: + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - 0] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg + elif i == 0 and j != 0 and j != nyf - 1: + seg = seg[margin : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg + elif i == nxf - 1 and j != 0 and j != nyf - 1: + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - 0] + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg + elif i != 0 and i != nxf - 1 and j == 0: + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - margin] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg + elif i != 0 and i != nxf - 1 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin, :] = seg + else: + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg + prediction_true = prediction_true.astype(int) + prediction_true = cv2.resize(prediction_true, (self.img_org.shape[1], self.img_org.shape[0]), interpolation=cv2.INTER_NEAREST) + return prediction_true + + else: + + img=cv2.imread(self.image) + self.img_org = np.copy(img) + + width=self.img_width + height=self.img_height + + img=img/255.0 + img=self.resize_image(img,self.img_height,self.img_width) + + + label_p_pred=self.model.predict( + img.reshape(1,img.shape[0],img.shape[1],img.shape[2])) + + if self.task == 'enhancement': + seg = label_p_pred[0, :, :, :] + seg = seg * 255 + elif self.task == 'segmentation' or self.task == 'binarization': + seg = np.argmax(label_p_pred, axis=3)[0] + seg = np.repeat(seg[:, :, np.newaxis], 3, axis=2) + + prediction_true = seg.astype(int) + + prediction_true = cv2.resize(prediction_true, (self.img_org.shape[1], self.img_org.shape[0]), interpolation=cv2.INTER_NEAREST) + return prediction_true + + + + def run(self): + res=self.predict() + if self.task == 'classification': + pass + else: + img_seg_overlayed = self.visualize_model_output(res, self.img_org, self.task) + cv2.imwrite('./test.png',img_seg_overlayed) + ##if self.save!=None: + ##img=np.repeat(res[:, :, np.newaxis]*255, 3, axis=2) + ##cv2.imwrite(self.save,img) + + ###if self.ground_truth!=None: + ###gt_img=cv2.imread(self.ground_truth) + ###self.IoU(gt_img[:,:,0],res) + ##plt.imshow(res) + ##plt.show() + +@click.command() +@click.option( + "--image", + "-i", + help="image filename", + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--patches/--no-patches", + "-p/-nop", + is_flag=True, + help="if this parameter set to true, this tool will try to do inference in patches.", +) +@click.option( + "--save", + "-s", + help="save prediction as a png file in current folder.", +) +@click.option( + "--model", + "-m", + help="directory of models", + type=click.Path(exists=True, file_okay=False), + required=True, +) +@click.option( + "--ground_truth/--no-ground_truth", + "-gt/-nogt", + is_flag=True, + help="ground truth directory if you want to see the iou of prediction.", +) +@click.option( + "--model_weights/--no-model_weights", + "-mw/-nomw", + is_flag=True, + help="previous model weights which are saved.", +) +def main(image, model, patches, save, ground_truth, model_weights): + + with open(os.path.join(model,'config.json')) as f: + config_params_model = json.load(f) + task = 'classification' + x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, model_weights) + x.run() + +if __name__=="__main__": + main() + + + + diff --git a/train/train.py b/train/train.py index 595debe..28363d2 100644 --- a/train/train.py +++ b/train/train.py @@ -69,7 +69,7 @@ def config_params(): flip_index = None # Flip image for augmentation. continue_training = False # Set to true if you would like to continue training an already trained a model. transformer_patchsize = None # Patch size of vision transformer patches. - num_patches_xy = None # Number of patches for vision transformer. + transformer_num_patches_xy = None # Number of patches for vision transformer. index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. @@ -77,6 +77,8 @@ def config_params(): data_is_provided = False # Only set this to true when you have already provided the input data and the train and eval data are in "dir_output". task = "segmentation" # This parameter defines task of model which can be segmentation, enhancement or classification. f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. + classification_classes_name = None # Dictionary of classification classes names. + backbone_type = None # As backbone we have 2 types of backbones. A vision transformer alongside a CNN and we call it "transformer" and only CNN called "nontransformer" @ex.automain @@ -89,12 +91,12 @@ def run(_config, n_classes, n_epochs, input_height, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, continue_training, transformer_patchsize, - num_patches_xy, model_name, flip_index, dir_eval, dir_output, - pretraining, learning_rate, task, f1_threshold_classification): + transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, + pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name): - if task == "segmentation" or "enhancement": + if task == "segmentation" or task == "enhancement": - num_patches = num_patches_xy[0]*num_patches_xy[1] + num_patches = transformer_num_patches_xy[0]*transformer_num_patches_xy[1] if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') dir_eval_flowing = os.path.join(dir_output, 'eval') @@ -191,14 +193,14 @@ def run(_config, n_classes, n_epochs, input_height, weights = weights / float(np.sum(weights)) if continue_training: - if model_name=='resnet50_unet': + if backbone_type=='nontransformer': if is_loss_soft_dice and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) if weighted_loss and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) if not is_loss_soft_dice and not weighted_loss: model = load_model(dir_of_start_model , compile=True) - elif model_name=='hybrid_transformer_cnn': + elif backbone_type=='transformer': if is_loss_soft_dice and task == "segmentation": model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) if weighted_loss and task == "segmentation": @@ -207,9 +209,9 @@ def run(_config, n_classes, n_epochs, input_height, model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) else: index_start = 0 - if model_name=='resnet50_unet': + if backbone_type=='nontransformer': model = resnet50_unet(n_classes, input_height, input_width, task, weight_decay, pretraining) - elif model_name=='hybrid_transformer_cnn': + elif backbone_type=='nontransformer': model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width, task, weight_decay, pretraining) #if you want to see the model structure just uncomment model summary. @@ -246,9 +248,9 @@ def run(_config, n_classes, n_epochs, input_height, validation_data=val_gen, validation_steps=1, epochs=1) - model.save(dir_output+'/'+'model_'+str(i)) + model.save(os.path.join(dir_output,'model_'+str(i))) - with open(dir_output+'/'+'model_'+str(i)+'/'+"config.json", "w") as fp: + with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON #os.system('rm -rf '+dir_train_flowing) @@ -257,14 +259,15 @@ def run(_config, n_classes, n_epochs, input_height, #model.save(dir_output+'/'+'model'+'.h5') elif task=='classification': configuration() - model = resnet50_classifier(n_classes, input_height, input_width,weight_decay,pretraining) + model = resnet50_classifier(n_classes, input_height, input_width, weight_decay, pretraining) opt_adam = Adam(learning_rate=0.001) model.compile(loss='categorical_crossentropy', optimizer = opt_adam,metrics=['accuracy']) - - testX, testY = generate_data_from_folder_evaluation(dir_eval, input_height, input_width, n_classes) + + list_classes = list(classification_classes_name.values()) + testX, testY = generate_data_from_folder_evaluation(dir_eval, input_height, input_width, n_classes, list_classes) #print(testY.shape, testY) @@ -280,7 +283,7 @@ def run(_config, n_classes, n_epochs, input_height, for i in range(n_epochs): #history = model.fit(trainX, trainY, epochs=1, batch_size=n_batch, validation_data=(testX, testY), verbose=2)#,class_weight=weights) - history = model.fit( generate_data_from_folder_training(dir_train, n_batch , input_height, input_width, n_classes), steps_per_epoch=num_rows / n_batch, verbose=0)#,class_weight=weights) + history = model.fit( generate_data_from_folder_training(dir_train, n_batch , input_height, input_width, n_classes, list_classes), steps_per_epoch=num_rows / n_batch, verbose=0)#,class_weight=weights) y_pr_class = [] for jj in range(testY.shape[0]): @@ -301,10 +304,6 @@ def run(_config, n_classes, n_epochs, input_height, score_best[0]=f1score model.save(os.path.join(dir_output,'model_best')) - - ##best_model=keras.models.clone_model(model) - ##best_model.build() - ##best_model.set_weights(model.get_weights()) if f1score > f1_threshold_classification: weights.append(model.get_weights() ) y_tot=y_tot+y_pr @@ -329,4 +328,9 @@ def run(_config, n_classes, n_epochs, input_height, ##best_model.save('model_taza.h5') model_weight_averaged.save(os.path.join(dir_output,'model_ens_avg')) + with open(os.path.join( os.path.join(dir_output,'model_ens_avg'), "config.json"), "w") as fp: + json.dump(_config, fp) # encode dict into JSON + + with open(os.path.join( os.path.join(dir_output,'model_best'), "config.json"), "w") as fp: + json.dump(_config, fp) # encode dict into JSON diff --git a/train/utils.py b/train/utils.py index 0c5a458..3a0375a 100644 --- a/train/utils.py +++ b/train/utils.py @@ -21,14 +21,14 @@ def return_number_of_total_training_data(path_classes): -def generate_data_from_folder_evaluation(path_classes, height, width, n_classes): - sub_classes = os.listdir(path_classes) +def generate_data_from_folder_evaluation(path_classes, height, width, n_classes, list_classes): + #sub_classes = os.listdir(path_classes) #n_classes = len(sub_classes) all_imgs = [] labels = [] - dicts =dict() - indexer= 0 - for sub_c in sub_classes: + #dicts =dict() + #indexer= 0 + for indexer, sub_c in enumerate(list_classes): sub_files = os.listdir(os.path.join(path_classes,sub_c )) sub_files = [os.path.join(path_classes,sub_c )+'/' + x for x in sub_files] #print( os.listdir(os.path.join(path_classes,sub_c )) ) @@ -37,8 +37,8 @@ def generate_data_from_folder_evaluation(path_classes, height, width, n_classes) #print( len(sub_labels) ) labels = labels + sub_labels - dicts[sub_c] = indexer - indexer +=1 + #dicts[sub_c] = indexer + #indexer +=1 categories = to_categorical(range(n_classes)).astype(np.int16)#[ [1 , 0, 0 , 0 , 0 , 0] , [0 , 1, 0 , 0 , 0 , 0] , [0 , 0, 1 , 0 , 0 , 0] , [0 , 0, 0 , 1 , 0 , 0] , [0 , 0, 0 , 0 , 1 , 0] , [0 , 0, 0 , 0 , 0 , 1] ] @@ -64,15 +64,15 @@ def generate_data_from_folder_evaluation(path_classes, height, width, n_classes) return ret_x/255., ret_y -def generate_data_from_folder_training(path_classes, batchsize, height, width, n_classes): - sub_classes = os.listdir(path_classes) - n_classes = len(sub_classes) +def generate_data_from_folder_training(path_classes, batchsize, height, width, n_classes, list_classes): + #sub_classes = os.listdir(path_classes) + #n_classes = len(sub_classes) all_imgs = [] labels = [] - dicts =dict() - indexer= 0 - for sub_c in sub_classes: + #dicts =dict() + #indexer= 0 + for indexer, sub_c in enumerate(list_classes): sub_files = os.listdir(os.path.join(path_classes,sub_c )) sub_files = [os.path.join(path_classes,sub_c )+'/' + x for x in sub_files] #print( os.listdir(os.path.join(path_classes,sub_c )) ) @@ -81,8 +81,8 @@ def generate_data_from_folder_training(path_classes, batchsize, height, width, n #print( len(sub_labels) ) labels = labels + sub_labels - dicts[sub_c] = indexer - indexer +=1 + #dicts[sub_c] = indexer + #indexer +=1 ids = np.array(range(len(labels))) random.shuffle(ids) From bc2ca7180208a780d2d34710b66bac379a096385 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 7 May 2024 16:24:12 +0200 Subject: [PATCH 043/374] modifications --- train/inference.py | 108 +++++++-------------------------------------- 1 file changed, 17 insertions(+), 91 deletions(-) diff --git a/train/inference.py b/train/inference.py index 6911bea..94e318d 100644 --- a/train/inference.py +++ b/train/inference.py @@ -1,25 +1,16 @@ -#! /usr/bin/env python3 - -__version__= '1.0' - -import argparse import sys import os import numpy as np import warnings -import xml.etree.ElementTree as et -import pandas as pd -from tqdm import tqdm -import csv import cv2 import seaborn as sns -import matplotlib.pyplot as plt from tensorflow.keras.models import load_model import tensorflow as tf from tensorflow.keras import backend as K from tensorflow.keras import layers import tensorflow.keras.losses from tensorflow.keras.layers import * +from models import * import click import json from tensorflow.python.keras import backend as tensorflow_backend @@ -37,70 +28,13 @@ __doc__=\ Tool to load model and predict for given image. """ -projection_dim = 64 -patch_size = 1 -num_patches =28*28 -class Patches(layers.Layer): - def __init__(self, **kwargs): - super(Patches, self).__init__() - self.patch_size = patch_size - - def call(self, images): - print(tf.shape(images)[1],'images') - print(self.patch_size,'self.patch_size') - batch_size = tf.shape(images)[0] - patches = tf.image.extract_patches( - images=images, - sizes=[1, self.patch_size, self.patch_size, 1], - strides=[1, self.patch_size, self.patch_size, 1], - rates=[1, 1, 1, 1], - padding="VALID", - ) - patch_dims = patches.shape[-1] - print(patches.shape,patch_dims,'patch_dims') - patches = tf.reshape(patches, [batch_size, -1, patch_dims]) - return patches - def get_config(self): - - config = super().get_config().copy() - config.update({ - 'patch_size': self.patch_size, - }) - return config - - -class PatchEncoder(layers.Layer): - def __init__(self, **kwargs): - super(PatchEncoder, self).__init__() - self.num_patches = num_patches - self.projection = layers.Dense(units=projection_dim) - self.position_embedding = layers.Embedding( - input_dim=num_patches, output_dim=projection_dim - ) - - def call(self, patch): - positions = tf.range(start=0, limit=self.num_patches, delta=1) - encoded = self.projection(patch) + self.position_embedding(positions) - return encoded - def get_config(self): - - config = super().get_config().copy() - config.update({ - 'num_patches': self.num_patches, - 'projection': self.projection, - 'position_embedding': self.position_embedding, - }) - return config - - class sbb_predict: - def __init__(self,image, model, task, config_params_model, patches='false',save='false', ground_truth=None,weights_dir=None ): + def __init__(self,image, model, task, config_params_model, patches, save, ground_truth): self.image=image self.patches=patches self.save=save self.model_dir=model self.ground_truth=ground_truth - self.weights_dir=weights_dir self.task=task self.config_params_model=config_params_model @@ -426,16 +360,12 @@ class sbb_predict: pass else: img_seg_overlayed = self.visualize_model_output(res, self.img_org, self.task) - cv2.imwrite('./test.png',img_seg_overlayed) - ##if self.save!=None: - ##img=np.repeat(res[:, :, np.newaxis]*255, 3, axis=2) - ##cv2.imwrite(self.save,img) - - ###if self.ground_truth!=None: - ###gt_img=cv2.imread(self.ground_truth) - ###self.IoU(gt_img[:,:,0],res) - ##plt.imshow(res) - ##plt.show() + if self.save: + cv2.imwrite(self.save,img_seg_overlayed) + + if self.ground_truth: + gt_img=cv2.imread(self.ground_truth) + self.IoU(gt_img[:,:,0],res[:,:,0]) @click.command() @click.option( @@ -463,23 +393,19 @@ class sbb_predict: required=True, ) @click.option( - "--ground_truth/--no-ground_truth", - "-gt/-nogt", - is_flag=True, + "--ground_truth", + "-gt", help="ground truth directory if you want to see the iou of prediction.", ) -@click.option( - "--model_weights/--no-model_weights", - "-mw/-nomw", - is_flag=True, - help="previous model weights which are saved.", -) -def main(image, model, patches, save, ground_truth, model_weights): - +def main(image, model, patches, save, ground_truth): with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) - task = 'classification' - x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, model_weights) + task = config_params_model['task'] + if task != 'classification': + if not save: + print("Error: You used one of segmentation or binarization task but not set -s, you need a filename to save visualized output with -s") + sys.exit(1) + x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth) x.run() if __name__=="__main__": From 241cb907cbb691988866011fdad5af12eb4986ae Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 8 May 2024 14:47:16 +0200 Subject: [PATCH 044/374] Update train.py avoid ensembling if no model weights met the threshold f1 score in the case of classification --- train/train.py | 46 +++++++++++++--------------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/train/train.py b/train/train.py index 28363d2..78974d3 100644 --- a/train/train.py +++ b/train/train.py @@ -268,36 +268,26 @@ def run(_config, n_classes, n_epochs, input_height, list_classes = list(classification_classes_name.values()) testX, testY = generate_data_from_folder_evaluation(dir_eval, input_height, input_width, n_classes, list_classes) - - #print(testY.shape, testY) y_tot=np.zeros((testX.shape[0],n_classes)) - indexer=0 score_best=[] score_best.append(0) num_rows = return_number_of_total_training_data(dir_train) - weights=[] for i in range(n_epochs): - #history = model.fit(trainX, trainY, epochs=1, batch_size=n_batch, validation_data=(testX, testY), verbose=2)#,class_weight=weights) - history = model.fit( generate_data_from_folder_training(dir_train, n_batch , input_height, input_width, n_classes, list_classes), steps_per_epoch=num_rows / n_batch, verbose=0)#,class_weight=weights) + history = model.fit( generate_data_from_folder_training(dir_train, n_batch , input_height, input_width, n_classes, list_classes), steps_per_epoch=num_rows / n_batch, verbose=1)#,class_weight=weights) y_pr_class = [] for jj in range(testY.shape[0]): y_pr=model.predict(testX[jj,:,:,:].reshape(1,input_height,input_width,3), verbose=0) y_pr_ind= np.argmax(y_pr,axis=1) - #print(y_pr_ind, 'y_pr_ind') y_pr_class.append(y_pr_ind) - y_pr_class = np.array(y_pr_class) - #model.save('./models_save/model_'+str(i)+'.h5') - #y_pr_class=np.argmax(y_pr,axis=1) f1score=f1_score(np.argmax(testY,axis=1), y_pr_class, average='macro') - print(i,f1score) if f1score>score_best[0]: @@ -306,30 +296,20 @@ def run(_config, n_classes, n_epochs, input_height, if f1score > f1_threshold_classification: weights.append(model.get_weights() ) - y_tot=y_tot+y_pr - indexer+=1 - y_tot=y_tot/float(indexer) - - new_weights=list() - - for weights_list_tuple in zip(*weights): - new_weights.append( [np.array(weights_).mean(axis=0) for weights_ in zip(*weights_list_tuple)] ) - - new_weights = [np.array(x) for x in new_weights] - - model_weight_averaged=tf.keras.models.clone_model(model) - - model_weight_averaged.set_weights(new_weights) - - #y_tot_end=np.argmax(y_tot,axis=1) - #print(f1_score(np.argmax(testY,axis=1), y_tot_end, average='macro')) - - ##best_model.save('model_taza.h5') - model_weight_averaged.save(os.path.join(dir_output,'model_ens_avg')) - with open(os.path.join( os.path.join(dir_output,'model_ens_avg'), "config.json"), "w") as fp: - json.dump(_config, fp) # encode dict into JSON + if len(weights) >= 1: + new_weights=list() + for weights_list_tuple in zip(*weights): + new_weights.append( [np.array(weights_).mean(axis=0) for weights_ in zip(*weights_list_tuple)] ) + + new_weights = [np.array(x) for x in new_weights] + model_weight_averaged=tf.keras.models.clone_model(model) + model_weight_averaged.set_weights(new_weights) + + model_weight_averaged.save(os.path.join(dir_output,'model_ens_avg')) + with open(os.path.join( os.path.join(dir_output,'model_ens_avg'), "config.json"), "w") as fp: + json.dump(_config, fp) # encode dict into JSON with open(os.path.join( os.path.join(dir_output,'model_best'), "config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON From d277ec4b31dd28a3da3d38e9f9fd37b5c3e17fb2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 12 May 2024 08:32:28 +0200 Subject: [PATCH 045/374] Update utils.py --- train/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/train/utils.py b/train/utils.py index 3a0375a..271d977 100644 --- a/train/utils.py +++ b/train/utils.py @@ -9,6 +9,7 @@ from tqdm import tqdm import imutils import math from tensorflow.keras.utils import to_categorical +from PIL import Image, ImageEnhance def return_number_of_total_training_data(path_classes): From d6a057ba702f31c03db0401ab97fcd1a444b89a0 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 16 May 2024 15:03:23 +0200 Subject: [PATCH 046/374] adding page xml to label generator --- train/pagexml2label.py | 1009 ++++++++++++++++++++++++++++++++++++++++ train/requirements.txt | 1 + 2 files changed, 1010 insertions(+) create mode 100644 train/pagexml2label.py diff --git a/train/pagexml2label.py b/train/pagexml2label.py new file mode 100644 index 0000000..715f99f --- /dev/null +++ b/train/pagexml2label.py @@ -0,0 +1,1009 @@ +import click +import sys +import os +import numpy as np +import warnings +import xml.etree.ElementTree as ET +from tqdm import tqdm +import cv2 +from shapely import geometry + +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + +__doc__=\ +""" +tool to extract 2d or 3d RGB images from page xml data. In former case output will be 1 +2D image array which each class has filled with a pixel value. In the case of 3D RGB image +each class will be defined with a RGB value and beside images a text file of classes also will be produced. +This classes.txt file is required for dhsegment tool. +""" +KERNEL = np.ones((5, 5), np.uint8) + +class pagexml2word: + def __init__(self,dir_in, out_dir,output_type,experiment): + self.dir=dir_in + self.output_dir=out_dir + self.output_type=output_type + self.experiment=experiment + + def get_content_of_dir(self): + """ + Listing all ground truth page xml files. All files are needed to have xml format. + """ + + gt_all=os.listdir(self.dir) + self.gt_list=[file for file in gt_all if file.split('.')[ len(file.split('.'))-1 ]=='xml' ] + + def return_parent_contours(self,contours, hierarchy): + contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1] + return contours_parent + def filter_contours_area_of_image_tables(self,image, contours, hierarchy, max_area, min_area): + found_polygons_early = list() + + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + # area = cv2.contourArea(c) + area = polygon.area + ##print(np.prod(thresh.shape[:2])) + # Check that polygon has area greater than minimal area + # print(hierarchy[0][jv][3],hierarchy ) + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : + # print(c[0][0][1]) + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.int32)) + jv += 1 + return found_polygons_early + + def return_contours_of_interested_region(self,region_pre_p, pixel, min_area=0.0002): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = self.return_parent_contours(contours_imgs, hierarchy) + contours_imgs = self.filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) + + return contours_imgs + + def get_images_of_ground_truth(self): + """ + Reading the page xml files and write the ground truth images into given output directory. + """ + for index in tqdm(range(len(self.gt_list))): + #try: + tree1 = ET.parse(self.dir+'/'+self.gt_list[index]) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + if self.experiment=='word': + region_tags=np.unique([x for x in alltags if x.endswith('Word')]) + co_word=[] + + for tag in region_tags: + if tag.endswith('}Word') or tag.endswith('}word'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_word.append(np.array(c_t_in)) + + img = np.zeros( (y_len,x_len, 3) ) + if self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_word, color=(1,1,1)) + elif self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_word, color=(255,0,0)) + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + + elif self.experiment=='glyph': + region_tags=np.unique([x for x in alltags if x.endswith('Glyph')]) + co_glyph=[] + + for tag in region_tags: + if tag.endswith('}Glyph') or tag.endswith('}glyph'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_glyph.append(np.array(c_t_in)) + + img = np.zeros( (y_len,x_len, 3) ) + if self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_glyph, color=(1,1,1)) + elif self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_glyph, color=(255,0,0)) + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + elif self.experiment=='textline': + region_tags=np.unique([x for x in alltags if x.endswith('TextLine')]) + co_line=[] + + for tag in region_tags: + if tag.endswith('}TextLine') or tag.endswith('}textline'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_line.append(np.array(c_t_in)) + + img = np.zeros( (y_len,x_len, 3) ) + if self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_line, color=(1,1,1)) + elif self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_line, color=(255,0,0)) + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + elif self.experiment=='layout_for_main_regions': + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + #print(region_tags) + co_text=[] + co_sep=[] + co_img=[] + #co_graphic=[] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_text.append(np.array(c_t_in)) + + elif tag.endswith('}ImageRegion') or tag.endswith('}GraphicRegion') or tag.endswith('}imageregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + + elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + + img = np.zeros( (y_len,x_len,3) ) + + if self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_text, color=(255,0,0)) + img_poly=cv2.fillPoly(img, pts =co_img, color=(0,255,0)) + img_poly=cv2.fillPoly(img, pts =co_sep, color=(0,0,255)) + ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) + elif self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_text, color=(1,1,1)) + img_poly=cv2.fillPoly(img, pts =co_img, color=(2,2,2)) + img_poly=cv2.fillPoly(img, pts =co_sep, color=(3,3,3)) + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + elif self.experiment=='textregion': + region_tags=np.unique([x for x in alltags if x.endswith('TextRegion')]) + co_textregion=[] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_textregion.append(np.array(c_t_in)) + + img = np.zeros( (y_len,x_len,3) ) + if self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_textregion, color=(255,0,0)) + elif self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_textregion, color=(1,1,1)) + + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + elif self.experiment=='layout': + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + + co_text_paragraph=[] + co_text_drop=[] + co_text_heading=[] + co_text_header=[] + co_text_marginalia=[] + co_text_catch=[] + co_text_page_number=[] + co_text_signature_mark=[] + co_sep=[] + co_img=[] + co_table=[] + co_graphic=[] + co_graphic_text_annotation=[] + co_graphic_decoration=[] + co_noise=[] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + for nn in root1.iter(tag): + c_t_in_drop=[] + c_t_in_paragraph=[] + c_t_in_heading=[] + c_t_in_header=[] + c_t_in_page_number=[] + c_t_in_signature_mark=[] + c_t_in_catch=[] + c_t_in_marginalia=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + + coords=bool(vv.attrib) + if coords: + #print('birda1') + p_h=vv.attrib['points'].split(' ') + + + + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + #if nn.attrib['type']=='paragraph': + + c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='heading': + c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + + c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + elif "type" in nn.attrib and nn.attrib['type']=='header': + c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='page-number': + + c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + + c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + else: + + c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + + break + else: + pass + + + if vv.tag==link+'Point': + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + #if nn.attrib['type']=='paragraph': + + c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='heading': + c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + + c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + elif "type" in nn.attrib and nn.attrib['type']=='header': + c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='page-number': + + c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + + c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + + else: + c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + + #c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + + if len(c_t_in_drop)>0: + co_text_drop.append(np.array(c_t_in_drop)) + if len(c_t_in_paragraph)>0: + co_text_paragraph.append(np.array(c_t_in_paragraph)) + if len(c_t_in_heading)>0: + co_text_heading.append(np.array(c_t_in_heading)) + + if len(c_t_in_header)>0: + co_text_header.append(np.array(c_t_in_header)) + if len(c_t_in_page_number)>0: + co_text_page_number.append(np.array(c_t_in_page_number)) + if len(c_t_in_catch)>0: + co_text_catch.append(np.array(c_t_in_catch)) + + if len(c_t_in_signature_mark)>0: + co_text_signature_mark.append(np.array(c_t_in_signature_mark)) + + if len(c_t_in_marginalia)>0: + co_text_marginalia.append(np.array(c_t_in_marginalia)) + + + elif tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + c_t_in_text_annotation=[] + c_t_in_decoration=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + #c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + #if nn.attrib['type']=='paragraph': + + c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='decoration': + + c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + else: + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + + break + else: + pass + + + if vv.tag==link+'Point': + + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + #if nn.attrib['type']=='paragraph': + + c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='decoration': + + c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + else: + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if len(c_t_in_text_annotation)>0: + co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) + if len(c_t_in_decoration)>0: + co_graphic_decoration.append(np.array(c_t_in_decoration)) + if len(c_t_in)>0: + co_graphic.append(np.array(c_t_in)) + + + + elif tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + + elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + + elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + + elif tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_noise.append(np.array(c_t_in)) + + + img = np.zeros( (y_len,x_len,3) ) + + if self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(255,0,0)) + + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(255,125,0)) + img_poly=cv2.fillPoly(img, pts =co_text_header, color=(255,0,125)) + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(125,255,125)) + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(125,125,0)) + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(0,125,255)) + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(0,125,0)) + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(125,125,125)) + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(0,125,255)) + + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(125,0,125)) + img_poly=cv2.fillPoly(img, pts =co_img, color=(0,255,0)) + img_poly=cv2.fillPoly(img, pts =co_sep, color=(0,0,255)) + img_poly=cv2.fillPoly(img, pts =co_table, color=(0,255,255)) + img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) + img_poly=cv2.fillPoly(img, pts =co_noise, color=(255,0,255)) + elif self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(1,1,1)) + + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(2,2,2)) + img_poly=cv2.fillPoly(img, pts =co_text_header, color=(2,2,2)) + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(3,3,3)) + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(4,4,4)) + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(5,5,5)) + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(6,6,6)) + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(7,7,7)) + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(8,8,8)) + + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(9,9,9)) + img_poly=cv2.fillPoly(img, pts =co_img, color=(10,10,10)) + img_poly=cv2.fillPoly(img, pts =co_sep, color=(11,11,11)) + img_poly=cv2.fillPoly(img, pts =co_table, color=(12,12,12)) + img_poly=cv2.fillPoly(img, pts =co_graphic, color=(13,13,14)) + img_poly=cv2.fillPoly(img, pts =co_noise, color=(15,15,15)) + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + + elif self.experiment=='layout_for_main_regions_new_concept': + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + #print(region_tags) + co_text=[] + co_sep=[] + co_img=[] + co_drop = [] + co_graphic=[] + co_table = [] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + c_t_in_drop = [] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + else: + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + if len(c_t_in)>0: + co_text.append(np.array(c_t_in)) + if len(c_t_in_drop)>0: + co_drop.append(np.array(c_t_in_drop)) + + elif tag.endswith('}ImageRegion') or tag.endswith('}GraphicRegion') or tag.endswith('}imageregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + + elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + + img_boundary = np.zeros( (y_len,x_len) ) + + + co_text_eroded = [] + for con in co_text: + #try: + img_boundary_in = np.zeros( (y_len,x_len) ) + img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + #print('bidiahhhhaaa') + + + + #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica + img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=2) + + pixel = 1 + min_size = 0 + con_eroded = self.return_contours_of_interested_region(img_boundary_in,pixel, min_size ) + + try: + co_text_eroded.append(con_eroded[0]) + except: + co_text_eroded.append(con) + + img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=4) + #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) + + boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + img_boundary[:,:][boundary[:,:]==1] =1 + + + ###co_table_eroded = [] + ###for con in co_table: + ####try: + ###img_boundary_in = np.zeros( (y_len,x_len) ) + ###img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + ####print('bidiahhhhaaa') + + + + #####img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica + ###img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=2) + + ###pixel = 1 + ###min_size = 0 + ###con_eroded = self.return_contours_of_interested_region(img_boundary_in,pixel, min_size ) + + ###try: + ###co_table_eroded.append(con_eroded[0]) + ###except: + ###co_table_eroded.append(con) + + ###img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=4) + + ###boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + ###img_boundary[:,:][boundary[:,:]==1] =1 + #except: + #pass + + #for con in co_img: + #img_boundary_in = np.zeros( (y_len,x_len) ) + #img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=3) + + #boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + #img_boundary[:,:][boundary[:,:]==1] =1 + + + #for con in co_sep: + + #img_boundary_in = np.zeros( (y_len,x_len) ) + #img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=3) + + #boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + img_boundary[:,:][boundary[:,:]==1] =1 + for con in co_drop: + img_boundary_in = np.zeros( (y_len,x_len) ) + img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=3) + + boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + img_boundary[:,:][boundary[:,:]==1] =1 + + + img = np.zeros( (y_len,x_len,3) ) + + if self.output_type == '2d': + img_poly=cv2.fillPoly(img, pts =co_img, color=(2,2,2)) + + img_poly=cv2.fillPoly(img, pts =co_text_eroded, color=(1,1,1)) + ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(4,4,4)) + ###img_poly=cv2.fillPoly(img, pts =co_table, color=(1,1,1)) + + img_poly=cv2.fillPoly(img, pts =co_drop, color=(1,1,1)) + img_poly[:,:][img_boundary[:,:]==1] = 4 + img_poly=cv2.fillPoly(img, pts =co_sep, color=(3,3,3)) + elif self.output_type == '3d': + img_poly=cv2.fillPoly(img, pts =co_img, color=(0,255,0)) + img_poly=cv2.fillPoly(img, pts =co_text_eroded, color=(255,0,0)) + img_poly=cv2.fillPoly(img, pts =co_drop, color=(0,125,255)) + + img_poly[:,:,0][img_boundary[:,:]==1]=255 + img_poly[:,:,1][img_boundary[:,:]==1]=125 + img_poly[:,:,2][img_boundary[:,:]==1]=125 + + img_poly=cv2.fillPoly(img, pts =co_sep, color=(0,0,255)) + ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) + + #print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png') + try: + #print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png') + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + + + #except: + #pass + def run(self): + self.get_content_of_dir() + self.get_images_of_ground_truth() + + +@click.command() +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--dir_out", + "-do", + help="directory where ground truth images would be written", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--type_output", + "-to", + help="this defines how output should be. A 2d image array or a 3d image array encoded with RGB color. Just pass 2d or 3d. The file will be saved one directory up. 2D image array is 3d but only information of one channel would be enough since all channels have the same values.", +) +@click.option( + "--experiment", + "-exp", + help="experiment of ineterst. Word , textline , glyph and textregion are desired options.", +) + +def main(dir_xml,dir_out,type_output,experiment): + x=pagexml2word(dir_xml,dir_out,type_output,experiment) + x.run() +if __name__=="__main__": + main() + + + diff --git a/train/requirements.txt b/train/requirements.txt index 3e56438..efee9df 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -7,3 +7,4 @@ imutils numpy scipy scikit-learn +shapely From faeac997e15c3dd824a029e8e798fc3e7a262a8c Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 17 May 2024 09:10:13 +0200 Subject: [PATCH 047/374] page to label enable textline new concept --- train/pagexml2label.py | 73 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/train/pagexml2label.py b/train/pagexml2label.py index 715f99f..b094e9b 100644 --- a/train/pagexml2label.py +++ b/train/pagexml2label.py @@ -217,6 +217,79 @@ class pagexml2word: except: cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + elif self.experiment == 'textline_new_concept': + region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) + co_line = [] + + for tag in region_tags: + if tag.endswith('}TextLine') or tag.endswith('}textline'): + # print('sth') + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + # check the format of coords + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(np.float(vv.attrib['x'])), int(np.float(vv.attrib['y']))]) + sumi += 1 + # print(vv.tag,'in') + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_line.append(np.array(c_t_in)) + + img_boundary = np.zeros((y_len, x_len)) + co_textline_eroded = [] + for con in co_line: + # try: + img_boundary_in = np.zeros((y_len, x_len)) + img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + # print('bidiahhhhaaa') + + # img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica + img_boundary_in = cv2.erode(img_boundary_in[:, :], KERNEL, iterations=1) + + pixel = 1 + min_size = 0 + con_eroded = self.return_contours_of_interested_region(img_boundary_in, pixel, min_size) + + try: + co_textline_eroded.append(con_eroded[0]) + except: + co_textline_eroded.append(con) + + img_boundary_in_dilated = cv2.dilate(img_boundary_in[:, :], KERNEL, iterations=3) + # img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) + + boundary = img_boundary_in_dilated[:, :] - img_boundary_in[:, :] + + img_boundary[:, :][boundary[:, :] == 1] = 1 + + img = np.zeros((y_len, x_len, 3)) + if self.output_type == '2d': + img_poly = cv2.fillPoly(img, pts=co_textline_eroded, color=(1, 1, 1)) + img_poly[:, :][img_boundary[:, :] == 1] = 2 + elif self.output_type == '3d': + img_poly = cv2.fillPoly(img, pts=co_textline_eroded, color=(255, 0, 0)) + img_poly[:, :, 0][img_boundary[:, :] == 1] = 255 + img_poly[:, :, 1][img_boundary[:, :] == 1] = 125 + img_poly[:, :, 2][img_boundary[:, :] == 1] = 125 + + try: + cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('-')[1].split('.')[0] + '.png', + img_poly) + except: + cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('.')[0] + '.png', img_poly) + elif self.experiment=='layout_for_main_regions': region_tags=np.unique([x for x in alltags if x.endswith('Region')]) #print(region_tags) From b2085a1d01ec6a501a6f0752f492ab71f3015723 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 17 May 2024 09:08:25 +0200 Subject: [PATCH 048/374] update requirements --- train/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/train/requirements.txt b/train/requirements.txt index efee9df..d8f9003 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -8,3 +8,4 @@ numpy scipy scikit-learn shapely +click From f1c2913c0394dbb64a5464afc183d3600a222f6b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 22 May 2024 12:38:24 +0200 Subject: [PATCH 049/374] page2label with a dynamic layout --- train/custom_config_page2label.json | 6 + train/pagexml2label.py | 490 +++++++++++++++++++++++++++- 2 files changed, 479 insertions(+), 17 deletions(-) create mode 100644 train/custom_config_page2label.json diff --git a/train/custom_config_page2label.json b/train/custom_config_page2label.json new file mode 100644 index 0000000..75c4b96 --- /dev/null +++ b/train/custom_config_page2label.json @@ -0,0 +1,6 @@ +{ +"textregions":{"paragraph":1, "heading": 2, "header":2,"drop-capital": 3, "marginal":4 }, +"imageregion":5, +"separatorregion":6, +"graphicregions" :{"handwritten-annotation":7, "decoration": 8, "signature": 9, "stamp": 10} +} diff --git a/train/pagexml2label.py b/train/pagexml2label.py index b094e9b..6907e84 100644 --- a/train/pagexml2label.py +++ b/train/pagexml2label.py @@ -7,6 +7,7 @@ import xml.etree.ElementTree as ET from tqdm import tqdm import cv2 from shapely import geometry +import json with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -21,11 +22,12 @@ This classes.txt file is required for dhsegment tool. KERNEL = np.ones((5, 5), np.uint8) class pagexml2word: - def __init__(self,dir_in, out_dir,output_type,experiment): + def __init__(self,dir_in, out_dir,output_type,experiment,layout_config): self.dir=dir_in self.output_dir=out_dir self.output_type=output_type self.experiment=experiment + self.layout_config=layout_config def get_content_of_dir(self): """ @@ -77,7 +79,7 @@ class pagexml2word: return contours_imgs - def get_images_of_ground_truth(self): + def get_images_of_ground_truth(self, config_params): """ Reading the page xml files and write the ground truth images into given output directory. """ @@ -93,6 +95,445 @@ class pagexml2word: for jj in root1.iter(link+'Page'): y_len=int(jj.attrib['imageHeight']) x_len=int(jj.attrib['imageWidth']) + + if self.layout_config: + keys = list(config_params.keys()) + #values = config_params.values() + + if 'textregions' in keys: + types_text_dict = config_params['textregions'] + types_text = list(types_text_dict.keys()) + types_text_label = list(types_text_dict.values()) + if 'graphicregions' in keys: + types_graphic_dict = config_params['graphicregions'] + types_graphic = list(types_graphic_dict.keys()) + types_graphic_label = list(types_graphic_dict.values()) + + + types_text_label_rgb = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (0,125,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,255), (0,255,125)] + + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + + co_text_paragraph=[] + co_text_drop=[] + co_text_heading=[] + co_text_header=[] + co_text_marginalia=[] + co_text_catch=[] + co_text_page_number=[] + co_text_signature_mark=[] + co_sep=[] + co_img=[] + co_table=[] + co_graphic_signature=[] + co_graphic_text_annotation=[] + co_graphic_decoration=[] + co_graphic_stamp=[] + co_noise=[] + + for tag in region_tags: + if 'textregions' in keys: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + for nn in root1.iter(tag): + c_t_in_drop=[] + c_t_in_paragraph=[] + c_t_in_heading=[] + c_t_in_header=[] + c_t_in_page_number=[] + c_t_in_signature_mark=[] + c_t_in_catch=[] + c_t_in_marginalia=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + + coords=bool(vv.attrib) + if coords: + #print('birda1') + p_h=vv.attrib['points'].split(' ') + + if "drop-capital" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "heading" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='heading': + c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "signature-mark" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='signature-mark': + c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "header" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='header': + c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "catch-word" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "page-number" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='page-number': + c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "marginalia" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='marginalia': + c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "paragraph" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='paragraph': + c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + break + else: + pass + + + if vv.tag==link+'Point': + if "drop-capital" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "heading" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='heading': + c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "signature-mark" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='signature-mark': + c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "header" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='header': + c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "catch-word" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "page-number" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='page-number': + c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "marginalia" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='marginalia': + c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "paragraph" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='paragraph': + c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif vv.tag!=link+'Point' and sumi>=1: + break + + if len(c_t_in_drop)>0: + co_text_drop.append(np.array(c_t_in_drop)) + if len(c_t_in_paragraph)>0: + co_text_paragraph.append(np.array(c_t_in_paragraph)) + if len(c_t_in_heading)>0: + co_text_heading.append(np.array(c_t_in_heading)) + + if len(c_t_in_header)>0: + co_text_header.append(np.array(c_t_in_header)) + if len(c_t_in_page_number)>0: + co_text_page_number.append(np.array(c_t_in_page_number)) + if len(c_t_in_catch)>0: + co_text_catch.append(np.array(c_t_in_catch)) + + if len(c_t_in_signature_mark)>0: + co_text_signature_mark.append(np.array(c_t_in_signature_mark)) + + if len(c_t_in_marginalia)>0: + co_text_marginalia.append(np.array(c_t_in_marginalia)) + + + if 'graphicregions' in keys: + if tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in_stamp=[] + c_t_in_text_annotation=[] + c_t_in_decoration=[] + c_t_in_signature=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + if "handwritten-annotation" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "decoration" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='decoration': + c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "stamp" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='stamp': + c_t_in_stamp.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "signature" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='signature': + c_t_in_signature.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + + break + else: + pass + + + if vv.tag==link+'Point': + if "handwritten-annotation" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "decoration" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='decoration': + c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "stamp" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='stamp': + c_t_in_stamp.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "signature" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='signature': + c_t_in_signature.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if len(c_t_in_text_annotation)>0: + co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) + if len(c_t_in_decoration)>0: + co_graphic_decoration.append(np.array(c_t_in_decoration)) + if len(c_t_in_stamp)>0: + co_graphic_stamp.append(np.array(c_t_in_stamp)) + if len(c_t_in_signature)>0: + co_graphic_signature.append(np.array(c_t_in_signature)) + + if 'imageregion' in keys: + if tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + + + if 'separatorregion' in keys: + if tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + + if 'tableregion' in keys: + if tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + + if 'noiseregion' in keys: + if tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_noise.append(np.array(c_t_in)) + + img = np.zeros( (y_len,x_len,3) ) + + if self.output_type == '3d': + + if 'graphicregions' in keys: + if "handwritten-annotation" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=types_text_label_rgb[ config_params['graphicregions']['handwritten-annotation']]) + if "signature" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=types_text_label_rgb[ config_params['graphicregions']['signature']]) + if "decoration" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=types_text_label_rgb[ config_params['graphicregions']['decoration']]) + if "stamp" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=types_text_label_rgb[ config_params['graphicregions']['stamp']]) + + if 'imageregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_img, color=types_text_label_rgb[ config_params['imageregion']]) + if 'separatorregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_sep, color=types_text_label_rgb[ config_params['separatorregion']]) + if 'tableregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_table, color=types_text_label_rgb[ config_params['tableregion']]) + if 'noiseregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_noise, color=types_text_label_rgb[ config_params['noiseregion']]) + + if 'textregions' in keys: + if "paragraph" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=types_text_label_rgb[ config_params['textregions']['paragraph']]) + if "heading" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=types_text_label_rgb[ config_params['textregions']['heading']]) + if "header" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_header, color=types_text_label_rgb[ config_params['textregions']['header']]) + if "catch-word" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=types_text_label_rgb[ config_params['textregions']['catch-word']]) + if "signature-mark" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=types_text_label_rgb[ config_params['textregions']['signature-mark']]) + if "page-number" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=types_text_label_rgb[ config_params['textregions']['page-number']]) + if "marginalia" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=types_text_label_rgb[ config_params['textregions']['marginalia']]) + if "drop-capital" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=types_text_label_rgb[ config_params['textregions']['drop-capital']]) + + elif self.output_type == '2d': + if 'graphicregions' in keys: + if "handwritten-annotation" in types_graphic: + color_label = config_params['graphicregions']['handwritten-annotation'] + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(color_label,color_label,color_label)) + if "signature" in types_graphic: + color_label = config_params['graphicregions']['signature'] + img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=(color_label,color_label,color_label)) + if "decoration" in types_graphic: + color_label = config_params['graphicregions']['decoration'] + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(color_label,color_label,color_label)) + if "stamp" in types_graphic: + color_label = config_params['graphicregions']['stamp'] + img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=(color_label,color_label,color_label)) + + if 'imageregion' in keys: + color_label = config_params['imageregion'] + img_poly=cv2.fillPoly(img, pts =co_img, color=(color_label,color_label,color_label)) + if 'separatorregion' in keys: + color_label = config_params['separatorregion'] + img_poly=cv2.fillPoly(img, pts =co_sep, color=(color_label,color_label,color_label)) + if 'tableregion' in keys: + color_label = config_params['tableregion'] + img_poly=cv2.fillPoly(img, pts =co_table, color=(color_label,color_label,color_label)) + if 'noiseregion' in keys: + color_label = config_params['noiseregion'] + img_poly=cv2.fillPoly(img, pts =co_noise, color=(color_label,color_label,color_label)) + + if 'textregions' in keys: + if "paragraph" in types_text: + color_label = config_params['textregions']['paragraph'] + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(color_label,color_label,color_label)) + if "heading" in types_text: + color_label = config_params['textregions']['heading'] + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(color_label,color_label,color_label)) + if "header" in types_text: + color_label = config_params['textregions']['header'] + img_poly=cv2.fillPoly(img, pts =co_text_header, color=(color_label,color_label,color_label)) + if "catch-word" in types_text: + color_label = config_params['textregions']['catch-word'] + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(color_label,color_label,color_label)) + if "signature-mark" in types_text: + color_label = config_params['textregions']['signature-mark'] + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(color_label,color_label,color_label)) + if "page-number" in types_text: + color_label = config_params['textregions']['page-number'] + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(color_label,color_label,color_label)) + if "marginalia" in types_text: + color_label = config_params['textregions']['marginalia'] + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(color_label,color_label,color_label)) + if "drop-capital" in types_text: + color_label = config_params['textregions']['drop-capital'] + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(color_label,color_label,color_label)) + + + + + try: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) + + + #print(values[0]) if self.experiment=='word': region_tags=np.unique([x for x in alltags if x.endswith('Word')]) co_word=[] @@ -302,6 +743,7 @@ class pagexml2word: if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): #print('sth') for nn in root1.iter(tag): + print(nn.attrib['type']) c_t_in=[] sumi=0 for vv in nn.iter(): @@ -373,20 +815,19 @@ class pagexml2word: elif vv.tag!=link+'Point' and sumi>=1: break co_sep.append(np.array(c_t_in)) - - - img = np.zeros( (y_len,x_len,3) ) + img_poly = np.zeros( (y_len,x_len,3) ) + if self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_text, color=(255,0,0)) - img_poly=cv2.fillPoly(img, pts =co_img, color=(0,255,0)) - img_poly=cv2.fillPoly(img, pts =co_sep, color=(0,0,255)) + img_poly=cv2.fillPoly(img_poly, pts =co_text, color=(255,0,0)) + img_poly=cv2.fillPoly(img_poly, pts =co_img, color=(0,255,0)) + img_poly=cv2.fillPoly(img_poly, pts =co_sep, color=(0,0,255)) ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) elif self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_text, color=(1,1,1)) - img_poly=cv2.fillPoly(img, pts =co_img, color=(2,2,2)) - img_poly=cv2.fillPoly(img, pts =co_sep, color=(3,3,3)) + img_poly=cv2.fillPoly(img_poly, pts =co_text, color=(1,1,1)) + img_poly=cv2.fillPoly(img_poly, pts =co_img, color=(2,2,2)) + img_poly=cv2.fillPoly(img_poly, pts =co_sep, color=(3,3,3)) try: cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) @@ -752,7 +1193,7 @@ class pagexml2word: img = np.zeros( (y_len,x_len,3) ) - + if self.output_type == '3d': img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(255,0,0)) @@ -1043,9 +1484,9 @@ class pagexml2word: #except: #pass - def run(self): + def run(self,config_params): self.get_content_of_dir() - self.get_images_of_ground_truth() + self.get_images_of_ground_truth(config_params) @click.command() @@ -1061,6 +1502,14 @@ class pagexml2word: help="directory where ground truth images would be written", type=click.Path(exists=True, file_okay=False), ) + +@click.option( + "--layout_config", + "-lc", + help="experiment of ineterst. Word , textline , glyph and textregion are desired options.", + type=click.Path(exists=True, dir_okay=False), +) + @click.option( "--type_output", "-to", @@ -1072,9 +1521,16 @@ class pagexml2word: help="experiment of ineterst. Word , textline , glyph and textregion are desired options.", ) -def main(dir_xml,dir_out,type_output,experiment): - x=pagexml2word(dir_xml,dir_out,type_output,experiment) - x.run() + +def main(dir_xml,dir_out,type_output,experiment,layout_config): + if layout_config: + with open(layout_config) as f: + config_params = json.load(f) + else: + print("passed") + config_params = None + x=pagexml2word(dir_xml,dir_out,type_output,experiment, layout_config) + x.run(config_params) if __name__=="__main__": main() From 47c6bf6b97db0e8ea9eb3e796cf9261ddaa2e4db Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 23 May 2024 11:14:14 +0200 Subject: [PATCH 050/374] dynamic layout decorated with artificial class on text elements boundry --- train/custom_config_page2label.json | 6 +- train/pagexml2label.py | 117 +++++++++++++++++++++++----- 2 files changed, 103 insertions(+), 20 deletions(-) diff --git a/train/custom_config_page2label.json b/train/custom_config_page2label.json index 75c4b96..85b5d7e 100644 --- a/train/custom_config_page2label.json +++ b/train/custom_config_page2label.json @@ -1,6 +1,8 @@ { -"textregions":{"paragraph":1, "heading": 2, "header":2,"drop-capital": 3, "marginal":4 }, +"textregions":{"paragraph":1, "heading": 2, "header":2,"drop-capital": 3, "marginalia":4 ,"page-number":1 , "catch-word":1 }, "imageregion":5, "separatorregion":6, -"graphicregions" :{"handwritten-annotation":7, "decoration": 8, "signature": 9, "stamp": 10} +"graphicregions" :{"handwritten-annotation":7, "decoration": 8, "signature": 9, "stamp": 10}, +"artificial_class_on_boundry": ["paragraph","header", "heading", "marginalia", "page-number", "catch-word", "drop-capital"], +"artificial_class_label":11 } diff --git a/train/pagexml2label.py b/train/pagexml2label.py index 6907e84..5311c24 100644 --- a/train/pagexml2label.py +++ b/train/pagexml2label.py @@ -78,7 +78,37 @@ class pagexml2word: contours_imgs = self.filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) return contours_imgs + def update_region_contours(self, co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len): + co_text_eroded = [] + for con in co_text: + #try: + img_boundary_in = np.zeros( (y_len,x_len) ) + img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + #print('bidiahhhhaaa') + + + + #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica + if erosion_rate > 0: + img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=erosion_rate) + + pixel = 1 + min_size = 0 + con_eroded = self.return_contours_of_interested_region(img_boundary_in,pixel, min_size ) + + try: + co_text_eroded.append(con_eroded[0]) + except: + co_text_eroded.append(con) + + img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_rate) + #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) + + boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + img_boundary[:,:][boundary[:,:]==1] =1 + return co_text_eroded, img_boundary def get_images_of_ground_truth(self, config_params): """ Reading the page xml files and write the ground truth images into given output directory. @@ -98,6 +128,10 @@ class pagexml2word: if self.layout_config: keys = list(config_params.keys()) + if "artificial_class_on_boundry" in keys: + elements_with_artificial_class = list(config_params['artificial_class_on_boundry']) + artificial_class_rgb_color = (255,255,0) + artificial_class_label = config_params['artificial_class_label'] #values = config_params.values() if 'textregions' in keys: @@ -110,7 +144,7 @@ class pagexml2word: types_graphic_label = list(types_graphic_dict.values()) - types_text_label_rgb = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (0,125,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,255), (0,255,125)] + labels_rgb_color = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (255,0,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,125), (0,255,125)] region_tags=np.unique([x for x in alltags if x.endswith('Region')]) @@ -429,46 +463,90 @@ class pagexml2word: break co_noise.append(np.array(c_t_in)) + if "artificial_class_on_boundry" in keys: + img_boundary = np.zeros( (y_len,x_len) ) + if "paragraph" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_paragraph, img_boundary = self.update_region_contours(co_text_paragraph, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "drop-capital" in elements_with_artificial_class: + erosion_rate = 0 + dilation_rate = 4 + co_text_drop, img_boundary = self.update_region_contours(co_text_drop, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "catch-word" in elements_with_artificial_class: + erosion_rate = 0 + dilation_rate = 4 + co_text_catch, img_boundary = self.update_region_contours(co_text_catch, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "page-number" in elements_with_artificial_class: + erosion_rate = 0 + dilation_rate = 4 + co_text_page_number, img_boundary = self.update_region_contours(co_text_page_number, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "header" in elements_with_artificial_class: + erosion_rate = 1 + dilation_rate = 4 + co_text_header, img_boundary = self.update_region_contours(co_text_header, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "heading" in elements_with_artificial_class: + erosion_rate = 1 + dilation_rate = 4 + co_text_heading, img_boundary = self.update_region_contours(co_text_heading, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "signature-mark" in elements_with_artificial_class: + erosion_rate = 1 + dilation_rate = 4 + co_text_signature_mark, img_boundary = self.update_region_contours(co_text_signature_mark, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "marginalia" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_marginalia, img_boundary = self.update_region_contours(co_text_marginalia, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + + img = np.zeros( (y_len,x_len,3) ) if self.output_type == '3d': if 'graphicregions' in keys: if "handwritten-annotation" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=types_text_label_rgb[ config_params['graphicregions']['handwritten-annotation']]) + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=labels_rgb_color[ config_params['graphicregions']['handwritten-annotation']]) if "signature" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=types_text_label_rgb[ config_params['graphicregions']['signature']]) + img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=labels_rgb_color[ config_params['graphicregions']['signature']]) if "decoration" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=types_text_label_rgb[ config_params['graphicregions']['decoration']]) + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=labels_rgb_color[ config_params['graphicregions']['decoration']]) if "stamp" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=types_text_label_rgb[ config_params['graphicregions']['stamp']]) + img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=labels_rgb_color[ config_params['graphicregions']['stamp']]) if 'imageregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_img, color=types_text_label_rgb[ config_params['imageregion']]) + img_poly=cv2.fillPoly(img, pts =co_img, color=labels_rgb_color[ config_params['imageregion']]) if 'separatorregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_sep, color=types_text_label_rgb[ config_params['separatorregion']]) + img_poly=cv2.fillPoly(img, pts =co_sep, color=labels_rgb_color[ config_params['separatorregion']]) if 'tableregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_table, color=types_text_label_rgb[ config_params['tableregion']]) + img_poly=cv2.fillPoly(img, pts =co_table, color=labels_rgb_color[ config_params['tableregion']]) if 'noiseregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_noise, color=types_text_label_rgb[ config_params['noiseregion']]) + img_poly=cv2.fillPoly(img, pts =co_noise, color=labels_rgb_color[ config_params['noiseregion']]) if 'textregions' in keys: if "paragraph" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=types_text_label_rgb[ config_params['textregions']['paragraph']]) + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=labels_rgb_color[ config_params['textregions']['paragraph']]) if "heading" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=types_text_label_rgb[ config_params['textregions']['heading']]) + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=labels_rgb_color[ config_params['textregions']['heading']]) if "header" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_header, color=types_text_label_rgb[ config_params['textregions']['header']]) + img_poly=cv2.fillPoly(img, pts =co_text_header, color=labels_rgb_color[ config_params['textregions']['header']]) if "catch-word" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=types_text_label_rgb[ config_params['textregions']['catch-word']]) + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=labels_rgb_color[ config_params['textregions']['catch-word']]) if "signature-mark" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=types_text_label_rgb[ config_params['textregions']['signature-mark']]) + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=labels_rgb_color[ config_params['textregions']['signature-mark']]) if "page-number" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=types_text_label_rgb[ config_params['textregions']['page-number']]) + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=labels_rgb_color[ config_params['textregions']['page-number']]) if "marginalia" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=types_text_label_rgb[ config_params['textregions']['marginalia']]) + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=labels_rgb_color[ config_params['textregions']['marginalia']]) if "drop-capital" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=types_text_label_rgb[ config_params['textregions']['drop-capital']]) + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=labels_rgb_color[ config_params['textregions']['drop-capital']]) + + if "artificial_class_on_boundry" in keys: + img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] + img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] + img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + + + elif self.output_type == '2d': if 'graphicregions' in keys: @@ -523,6 +601,9 @@ class pagexml2word: if "drop-capital" in types_text: color_label = config_params['textregions']['drop-capital'] img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(color_label,color_label,color_label)) + + if "artificial_class_on_boundry" in keys: + img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label @@ -1506,7 +1587,7 @@ class pagexml2word: @click.option( "--layout_config", "-lc", - help="experiment of ineterst. Word , textline , glyph and textregion are desired options.", + help="config file of prefered layout.", type=click.Path(exists=True, dir_okay=False), ) From 348d323c7cd98c53bfdbde37c517c5217db14f11 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 23 May 2024 15:43:31 +0200 Subject: [PATCH 051/374] missing text types are added --- train/custom_config_page2label.json | 12 ++++---- train/pagexml2label.py | 48 ++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/train/custom_config_page2label.json b/train/custom_config_page2label.json index 85b5d7e..254f4df 100644 --- a/train/custom_config_page2label.json +++ b/train/custom_config_page2label.json @@ -1,8 +1,8 @@ { -"textregions":{"paragraph":1, "heading": 2, "header":2,"drop-capital": 3, "marginalia":4 ,"page-number":1 , "catch-word":1 }, -"imageregion":5, -"separatorregion":6, -"graphicregions" :{"handwritten-annotation":7, "decoration": 8, "signature": 9, "stamp": 10}, -"artificial_class_on_boundry": ["paragraph","header", "heading", "marginalia", "page-number", "catch-word", "drop-capital"], -"artificial_class_label":11 +"textregions":{"paragraph":1, "heading": 1, "header":1,"drop-capital": 1, "marginalia":1 ,"page-number":1 , "catch-word":1 ,"footnote": 1, "footnote-continued": 1}, +"imageregion":2, +"separatorregion":3, +"graphicregions" :{"handwritten-annotation":2, "decoration": 2, "signature": 2, "stamp": 2}, +"artificial_class_on_boundry": ["paragraph","header", "heading", "marginalia", "page-number", "catch-word", "drop-capital","footnote", "footnote-continued"], +"artificial_class_label":4 } diff --git a/train/pagexml2label.py b/train/pagexml2label.py index 5311c24..63b7acf 100644 --- a/train/pagexml2label.py +++ b/train/pagexml2label.py @@ -113,6 +113,7 @@ class pagexml2word: """ Reading the page xml files and write the ground truth images into given output directory. """ + ## to do: add footnote to text regions for index in tqdm(range(len(self.gt_list))): #try: tree1 = ET.parse(self.dir+'/'+self.gt_list[index]) @@ -144,11 +145,13 @@ class pagexml2word: types_graphic_label = list(types_graphic_dict.values()) - labels_rgb_color = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (255,0,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,125), (0,255,125)] + labels_rgb_color = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (255,0,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,125), (0,255,125), (255,125,255), (125,255,0)] region_tags=np.unique([x for x in alltags if x.endswith('Region')]) co_text_paragraph=[] + co_text_footnote=[] + co_text_footnote_con=[] co_text_drop=[] co_text_heading=[] co_text_header=[] @@ -177,6 +180,8 @@ class pagexml2word: c_t_in_signature_mark=[] c_t_in_catch=[] c_t_in_marginalia=[] + c_t_in_footnote=[] + c_t_in_footnote_con=[] sumi=0 for vv in nn.iter(): # check the format of coords @@ -190,6 +195,14 @@ class pagexml2word: if "drop-capital" in types_text: if "type" in nn.attrib and nn.attrib['type']=='drop-capital': c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "footnote" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote': + c_t_in_footnote.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "footnote-continued" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': + c_t_in_footnote_con.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) if "heading" in types_text: if "type" in nn.attrib and nn.attrib['type']=='heading': @@ -231,6 +244,16 @@ class pagexml2word: c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) sumi+=1 + if "footnote" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote': + c_t_in_footnote.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "footnote-continued" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': + c_t_in_footnote_con.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + if "heading" in types_text: if "type" in nn.attrib and nn.attrib['type']=='heading': c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) @@ -272,6 +295,10 @@ class pagexml2word: if len(c_t_in_drop)>0: co_text_drop.append(np.array(c_t_in_drop)) + if len(c_t_in_footnote_con)>0: + co_text_footnote_con.append(np.array(c_t_in_footnote_con)) + if len(c_t_in_footnote)>0: + co_text_footnote.append(np.array(c_t_in_footnote)) if len(c_t_in_paragraph)>0: co_text_paragraph.append(np.array(c_t_in_paragraph)) if len(c_t_in_heading)>0: @@ -497,6 +524,15 @@ class pagexml2word: erosion_rate = 2 dilation_rate = 4 co_text_marginalia, img_boundary = self.update_region_contours(co_text_marginalia, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "footnote" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_footnote, img_boundary = self.update_region_contours(co_text_footnote, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "footnote-continued" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_footnote_con, img_boundary = self.update_region_contours(co_text_footnote_con, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + img = np.zeros( (y_len,x_len,3) ) @@ -525,6 +561,10 @@ class pagexml2word: if 'textregions' in keys: if "paragraph" in types_text: img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=labels_rgb_color[ config_params['textregions']['paragraph']]) + if "footnote" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=labels_rgb_color[ config_params['textregions']['footnote']]) + if "footnote-continued" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=labels_rgb_color[ config_params['textregions']['footnote-continued']]) if "heading" in types_text: img_poly=cv2.fillPoly(img, pts =co_text_heading, color=labels_rgb_color[ config_params['textregions']['heading']]) if "header" in types_text: @@ -580,6 +620,12 @@ class pagexml2word: if "paragraph" in types_text: color_label = config_params['textregions']['paragraph'] img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(color_label,color_label,color_label)) + if "footnote" in types_text: + color_label = config_params['textregions']['footnote'] + img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=(color_label,color_label,color_label)) + if "footnote-continued" in types_text: + color_label = config_params['textregions']['footnote-continued'] + img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=(color_label,color_label,color_label)) if "heading" in types_text: color_label = config_params['textregions']['heading'] img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(color_label,color_label,color_label)) From a83d53c27d09c962c54f441e225c70fbd820900b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 23 May 2024 17:14:31 +0200 Subject: [PATCH 052/374] use cases like textline, word and glyph are added --- train/custom_config_page2label.json | 11 +- train/pagexml2label.py | 1055 +++------------------------ 2 files changed, 93 insertions(+), 973 deletions(-) diff --git a/train/custom_config_page2label.json b/train/custom_config_page2label.json index 254f4df..d6320fa 100644 --- a/train/custom_config_page2label.json +++ b/train/custom_config_page2label.json @@ -1,8 +1,9 @@ { -"textregions":{"paragraph":1, "heading": 1, "header":1,"drop-capital": 1, "marginalia":1 ,"page-number":1 , "catch-word":1 ,"footnote": 1, "footnote-continued": 1}, -"imageregion":2, -"separatorregion":3, -"graphicregions" :{"handwritten-annotation":2, "decoration": 2, "signature": 2, "stamp": 2}, +"use_case": "layout", +"textregions":{"paragraph":1, "heading": 2, "header":2,"drop-capital": 3, "marginalia":4 ,"page-number":1 , "catch-word":1 ,"footnote": 1, "footnote-continued": 1}, +"imageregion":5, +"separatorregion":6, +"graphicregions" :{"handwritten-annotation":5, "decoration": 5, "signature": 5, "stamp": 5}, "artificial_class_on_boundry": ["paragraph","header", "heading", "marginalia", "page-number", "catch-word", "drop-capital","footnote", "footnote-continued"], -"artificial_class_label":4 +"artificial_class_label":7 } diff --git a/train/pagexml2label.py b/train/pagexml2label.py index 63b7acf..16cda8b 100644 --- a/train/pagexml2label.py +++ b/train/pagexml2label.py @@ -21,13 +21,12 @@ This classes.txt file is required for dhsegment tool. """ KERNEL = np.ones((5, 5), np.uint8) -class pagexml2word: - def __init__(self,dir_in, out_dir,output_type,experiment,layout_config): +class pagexml2label: + def __init__(self,dir_in, out_dir,output_type,config): self.dir=dir_in self.output_dir=out_dir self.output_type=output_type - self.experiment=experiment - self.layout_config=layout_config + self.config=config def get_content_of_dir(self): """ @@ -127,7 +126,82 @@ class pagexml2word: y_len=int(jj.attrib['imageHeight']) x_len=int(jj.attrib['imageWidth']) - if self.layout_config: + if self.config and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph'): + keys = list(config_params.keys()) + if "artificial_class_label" in keys: + artificial_class_rgb_color = (255,255,0) + artificial_class_label = config_params['artificial_class_label'] + + textline_rgb_color = (255, 0, 0) + + if config_params['use_case']=='textline': + region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) + elif config_params['use_case']=='word': + region_tags = np.unique([x for x in alltags if x.endswith('Word')]) + elif config_params['use_case']=='glyph': + region_tags = np.unique([x for x in alltags if x.endswith('Glyph')]) + co_use_case = [] + + for tag in region_tags: + if config_params['use_case']=='textline': + tag_endings = ['}TextLine','}textline'] + elif config_params['use_case']=='word': + tag_endings = ['}Word','}word'] + elif config_params['use_case']=='glyph': + tag_endings = ['}Glyph','}glyph'] + + if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + # check the format of coords + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(np.float(vv.attrib['x'])), int(np.float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_use_case.append(np.array(c_t_in)) + + + + if "artificial_class_label" in keys: + img_boundary = np.zeros((y_len, x_len)) + erosion_rate = 1 + dilation_rate = 3 + co_use_case, img_boundary = self.update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + + + img = np.zeros((y_len, x_len, 3)) + if self.output_type == '2d': + img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) + if "artificial_class_label" in keys: + img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label + elif self.output_type == '3d': + img_poly = cv2.fillPoly(img, pts=co_use_case, color=textline_rgb_color) + if "artificial_class_label" in keys: + img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] + img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] + img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + + try: + cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('-')[1].split('.')[0] + '.png', + img_poly) + except: + cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('.')[0] + '.png', img_poly) + + + if self.config and config_params['use_case']=='layout': keys = list(config_params.keys()) if "artificial_class_on_boundry" in keys: elements_with_artificial_class = list(config_params['artificial_class_on_boundry']) @@ -139,6 +213,7 @@ class pagexml2word: types_text_dict = config_params['textregions'] types_text = list(types_text_dict.keys()) types_text_label = list(types_text_dict.values()) + print(types_text) if 'graphicregions' in keys: types_graphic_dict = config_params['graphicregions'] types_graphic = list(types_graphic_dict.keys()) @@ -660,957 +735,6 @@ class pagexml2word: cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - #print(values[0]) - if self.experiment=='word': - region_tags=np.unique([x for x in alltags if x.endswith('Word')]) - co_word=[] - - for tag in region_tags: - if tag.endswith('}Word') or tag.endswith('}word'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_word.append(np.array(c_t_in)) - - img = np.zeros( (y_len,x_len, 3) ) - if self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_word, color=(1,1,1)) - elif self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_word, color=(255,0,0)) - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - - elif self.experiment=='glyph': - region_tags=np.unique([x for x in alltags if x.endswith('Glyph')]) - co_glyph=[] - - for tag in region_tags: - if tag.endswith('}Glyph') or tag.endswith('}glyph'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_glyph.append(np.array(c_t_in)) - - img = np.zeros( (y_len,x_len, 3) ) - if self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_glyph, color=(1,1,1)) - elif self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_glyph, color=(255,0,0)) - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - elif self.experiment=='textline': - region_tags=np.unique([x for x in alltags if x.endswith('TextLine')]) - co_line=[] - - for tag in region_tags: - if tag.endswith('}TextLine') or tag.endswith('}textline'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_line.append(np.array(c_t_in)) - - img = np.zeros( (y_len,x_len, 3) ) - if self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_line, color=(1,1,1)) - elif self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_line, color=(255,0,0)) - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - elif self.experiment == 'textline_new_concept': - region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) - co_line = [] - - for tag in region_tags: - if tag.endswith('}TextLine') or tag.endswith('}textline'): - # print('sth') - for nn in root1.iter(tag): - c_t_in = [] - sumi = 0 - for vv in nn.iter(): - # check the format of coords - if vv.tag == link + 'Coords': - coords = bool(vv.attrib) - if coords: - p_h = vv.attrib['points'].split(' ') - c_t_in.append( - np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) - break - else: - pass - - if vv.tag == link + 'Point': - c_t_in.append([int(np.float(vv.attrib['x'])), int(np.float(vv.attrib['y']))]) - sumi += 1 - # print(vv.tag,'in') - elif vv.tag != link + 'Point' and sumi >= 1: - break - co_line.append(np.array(c_t_in)) - - img_boundary = np.zeros((y_len, x_len)) - co_textline_eroded = [] - for con in co_line: - # try: - img_boundary_in = np.zeros((y_len, x_len)) - img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - # print('bidiahhhhaaa') - - # img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica - img_boundary_in = cv2.erode(img_boundary_in[:, :], KERNEL, iterations=1) - - pixel = 1 - min_size = 0 - con_eroded = self.return_contours_of_interested_region(img_boundary_in, pixel, min_size) - - try: - co_textline_eroded.append(con_eroded[0]) - except: - co_textline_eroded.append(con) - - img_boundary_in_dilated = cv2.dilate(img_boundary_in[:, :], KERNEL, iterations=3) - # img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) - - boundary = img_boundary_in_dilated[:, :] - img_boundary_in[:, :] - - img_boundary[:, :][boundary[:, :] == 1] = 1 - - img = np.zeros((y_len, x_len, 3)) - if self.output_type == '2d': - img_poly = cv2.fillPoly(img, pts=co_textline_eroded, color=(1, 1, 1)) - img_poly[:, :][img_boundary[:, :] == 1] = 2 - elif self.output_type == '3d': - img_poly = cv2.fillPoly(img, pts=co_textline_eroded, color=(255, 0, 0)) - img_poly[:, :, 0][img_boundary[:, :] == 1] = 255 - img_poly[:, :, 1][img_boundary[:, :] == 1] = 125 - img_poly[:, :, 2][img_boundary[:, :] == 1] = 125 - - try: - cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('-')[1].split('.')[0] + '.png', - img_poly) - except: - cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('.')[0] + '.png', img_poly) - - elif self.experiment=='layout_for_main_regions': - region_tags=np.unique([x for x in alltags if x.endswith('Region')]) - #print(region_tags) - co_text=[] - co_sep=[] - co_img=[] - #co_graphic=[] - - for tag in region_tags: - if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): - #print('sth') - for nn in root1.iter(tag): - print(nn.attrib['type']) - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_text.append(np.array(c_t_in)) - - elif tag.endswith('}ImageRegion') or tag.endswith('}GraphicRegion') or tag.endswith('}imageregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_img.append(np.array(c_t_in)) - - elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_sep.append(np.array(c_t_in)) - - img_poly = np.zeros( (y_len,x_len,3) ) - - - if self.output_type == '3d': - img_poly=cv2.fillPoly(img_poly, pts =co_text, color=(255,0,0)) - img_poly=cv2.fillPoly(img_poly, pts =co_img, color=(0,255,0)) - img_poly=cv2.fillPoly(img_poly, pts =co_sep, color=(0,0,255)) - ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) - elif self.output_type == '2d': - img_poly=cv2.fillPoly(img_poly, pts =co_text, color=(1,1,1)) - img_poly=cv2.fillPoly(img_poly, pts =co_img, color=(2,2,2)) - img_poly=cv2.fillPoly(img_poly, pts =co_sep, color=(3,3,3)) - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - elif self.experiment=='textregion': - region_tags=np.unique([x for x in alltags if x.endswith('TextRegion')]) - co_textregion=[] - - for tag in region_tags: - if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_textregion.append(np.array(c_t_in)) - - img = np.zeros( (y_len,x_len,3) ) - if self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_textregion, color=(255,0,0)) - elif self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_textregion, color=(1,1,1)) - - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - elif self.experiment=='layout': - region_tags=np.unique([x for x in alltags if x.endswith('Region')]) - - co_text_paragraph=[] - co_text_drop=[] - co_text_heading=[] - co_text_header=[] - co_text_marginalia=[] - co_text_catch=[] - co_text_page_number=[] - co_text_signature_mark=[] - co_sep=[] - co_img=[] - co_table=[] - co_graphic=[] - co_graphic_text_annotation=[] - co_graphic_decoration=[] - co_noise=[] - - for tag in region_tags: - if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): - for nn in root1.iter(tag): - c_t_in_drop=[] - c_t_in_paragraph=[] - c_t_in_heading=[] - c_t_in_header=[] - c_t_in_page_number=[] - c_t_in_signature_mark=[] - c_t_in_catch=[] - c_t_in_marginalia=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - - coords=bool(vv.attrib) - if coords: - #print('birda1') - p_h=vv.attrib['points'].split(' ') - - - - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - #if nn.attrib['type']=='paragraph': - - c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - elif "type" in nn.attrib and nn.attrib['type']=='heading': - c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - - elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': - - c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) - elif "type" in nn.attrib and nn.attrib['type']=='header': - c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - - elif "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - - elif "type" in nn.attrib and nn.attrib['type']=='page-number': - - c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) - - elif "type" in nn.attrib and nn.attrib['type']=='marginalia': - - c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) - else: - - c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) - - break - else: - pass - - - if vv.tag==link+'Point': - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - #if nn.attrib['type']=='paragraph': - - c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - elif "type" in nn.attrib and nn.attrib['type']=='heading': - c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - - elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': - - c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) - sumi+=1 - elif "type" in nn.attrib and nn.attrib['type']=='header': - c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - - elif "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - - elif "type" in nn.attrib and nn.attrib['type']=='page-number': - - c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) - sumi+=1 - - elif "type" in nn.attrib and nn.attrib['type']=='marginalia': - - c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) - sumi+=1 - - else: - c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) - sumi+=1 - - #c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - - if len(c_t_in_drop)>0: - co_text_drop.append(np.array(c_t_in_drop)) - if len(c_t_in_paragraph)>0: - co_text_paragraph.append(np.array(c_t_in_paragraph)) - if len(c_t_in_heading)>0: - co_text_heading.append(np.array(c_t_in_heading)) - - if len(c_t_in_header)>0: - co_text_header.append(np.array(c_t_in_header)) - if len(c_t_in_page_number)>0: - co_text_page_number.append(np.array(c_t_in_page_number)) - if len(c_t_in_catch)>0: - co_text_catch.append(np.array(c_t_in_catch)) - - if len(c_t_in_signature_mark)>0: - co_text_signature_mark.append(np.array(c_t_in_signature_mark)) - - if len(c_t_in_marginalia)>0: - co_text_marginalia.append(np.array(c_t_in_marginalia)) - - - elif tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - c_t_in_text_annotation=[] - c_t_in_decoration=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - #c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - #if nn.attrib['type']=='paragraph': - - c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - elif "type" in nn.attrib and nn.attrib['type']=='decoration': - - c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) - else: - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - - - break - else: - pass - - - if vv.tag==link+'Point': - - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - #if nn.attrib['type']=='paragraph': - - c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - elif "type" in nn.attrib and nn.attrib['type']=='decoration': - - c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) - sumi+=1 - else: - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if len(c_t_in_text_annotation)>0: - co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) - if len(c_t_in_decoration)>0: - co_graphic_decoration.append(np.array(c_t_in_decoration)) - if len(c_t_in)>0: - co_graphic.append(np.array(c_t_in)) - - - - elif tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_img.append(np.array(c_t_in)) - - elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_sep.append(np.array(c_t_in)) - - - - elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_table.append(np.array(c_t_in)) - - elif tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_noise.append(np.array(c_t_in)) - - - img = np.zeros( (y_len,x_len,3) ) - - if self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(255,0,0)) - - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(255,125,0)) - img_poly=cv2.fillPoly(img, pts =co_text_header, color=(255,0,125)) - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(125,255,125)) - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(125,125,0)) - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(0,125,255)) - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(0,125,0)) - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(125,125,125)) - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(0,125,255)) - - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(125,0,125)) - img_poly=cv2.fillPoly(img, pts =co_img, color=(0,255,0)) - img_poly=cv2.fillPoly(img, pts =co_sep, color=(0,0,255)) - img_poly=cv2.fillPoly(img, pts =co_table, color=(0,255,255)) - img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) - img_poly=cv2.fillPoly(img, pts =co_noise, color=(255,0,255)) - elif self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(1,1,1)) - - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(2,2,2)) - img_poly=cv2.fillPoly(img, pts =co_text_header, color=(2,2,2)) - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(3,3,3)) - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(4,4,4)) - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(5,5,5)) - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(6,6,6)) - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(7,7,7)) - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(8,8,8)) - - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(9,9,9)) - img_poly=cv2.fillPoly(img, pts =co_img, color=(10,10,10)) - img_poly=cv2.fillPoly(img, pts =co_sep, color=(11,11,11)) - img_poly=cv2.fillPoly(img, pts =co_table, color=(12,12,12)) - img_poly=cv2.fillPoly(img, pts =co_graphic, color=(13,13,14)) - img_poly=cv2.fillPoly(img, pts =co_noise, color=(15,15,15)) - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - - elif self.experiment=='layout_for_main_regions_new_concept': - region_tags=np.unique([x for x in alltags if x.endswith('Region')]) - #print(region_tags) - co_text=[] - co_sep=[] - co_img=[] - co_drop = [] - co_graphic=[] - co_table = [] - - for tag in region_tags: - if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - c_t_in_drop = [] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - else: - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - else: - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - if len(c_t_in)>0: - co_text.append(np.array(c_t_in)) - if len(c_t_in_drop)>0: - co_drop.append(np.array(c_t_in_drop)) - - elif tag.endswith('}ImageRegion') or tag.endswith('}GraphicRegion') or tag.endswith('}imageregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_img.append(np.array(c_t_in)) - - elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_sep.append(np.array(c_t_in)) - - elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_table.append(np.array(c_t_in)) - - img_boundary = np.zeros( (y_len,x_len) ) - - - co_text_eroded = [] - for con in co_text: - #try: - img_boundary_in = np.zeros( (y_len,x_len) ) - img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - #print('bidiahhhhaaa') - - - - #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica - img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=2) - - pixel = 1 - min_size = 0 - con_eroded = self.return_contours_of_interested_region(img_boundary_in,pixel, min_size ) - - try: - co_text_eroded.append(con_eroded[0]) - except: - co_text_eroded.append(con) - - img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=4) - #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) - - boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] - - img_boundary[:,:][boundary[:,:]==1] =1 - - - ###co_table_eroded = [] - ###for con in co_table: - ####try: - ###img_boundary_in = np.zeros( (y_len,x_len) ) - ###img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - ####print('bidiahhhhaaa') - - - - #####img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica - ###img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=2) - - ###pixel = 1 - ###min_size = 0 - ###con_eroded = self.return_contours_of_interested_region(img_boundary_in,pixel, min_size ) - - ###try: - ###co_table_eroded.append(con_eroded[0]) - ###except: - ###co_table_eroded.append(con) - - ###img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=4) - - ###boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] - - ###img_boundary[:,:][boundary[:,:]==1] =1 - #except: - #pass - - #for con in co_img: - #img_boundary_in = np.zeros( (y_len,x_len) ) - #img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=3) - - #boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] - - #img_boundary[:,:][boundary[:,:]==1] =1 - - - #for con in co_sep: - - #img_boundary_in = np.zeros( (y_len,x_len) ) - #img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=3) - - #boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] - - img_boundary[:,:][boundary[:,:]==1] =1 - for con in co_drop: - img_boundary_in = np.zeros( (y_len,x_len) ) - img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=3) - - boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] - - img_boundary[:,:][boundary[:,:]==1] =1 - - - img = np.zeros( (y_len,x_len,3) ) - - if self.output_type == '2d': - img_poly=cv2.fillPoly(img, pts =co_img, color=(2,2,2)) - - img_poly=cv2.fillPoly(img, pts =co_text_eroded, color=(1,1,1)) - ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(4,4,4)) - ###img_poly=cv2.fillPoly(img, pts =co_table, color=(1,1,1)) - - img_poly=cv2.fillPoly(img, pts =co_drop, color=(1,1,1)) - img_poly[:,:][img_boundary[:,:]==1] = 4 - img_poly=cv2.fillPoly(img, pts =co_sep, color=(3,3,3)) - elif self.output_type == '3d': - img_poly=cv2.fillPoly(img, pts =co_img, color=(0,255,0)) - img_poly=cv2.fillPoly(img, pts =co_text_eroded, color=(255,0,0)) - img_poly=cv2.fillPoly(img, pts =co_drop, color=(0,125,255)) - - img_poly[:,:,0][img_boundary[:,:]==1]=255 - img_poly[:,:,1][img_boundary[:,:]==1]=125 - img_poly[:,:,2][img_boundary[:,:]==1]=125 - - img_poly=cv2.fillPoly(img, pts =co_sep, color=(0,0,255)) - ##img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) - - #print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png') - try: - #print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png') - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - - - #except: - #pass def run(self,config_params): self.get_content_of_dir() self.get_images_of_ground_truth(config_params) @@ -1631,9 +755,9 @@ class pagexml2word: ) @click.option( - "--layout_config", - "-lc", - help="config file of prefered layout.", + "--config", + "-cfg", + help="config file of prefered layout or use case.", type=click.Path(exists=True, dir_okay=False), ) @@ -1642,21 +766,16 @@ class pagexml2word: "-to", help="this defines how output should be. A 2d image array or a 3d image array encoded with RGB color. Just pass 2d or 3d. The file will be saved one directory up. 2D image array is 3d but only information of one channel would be enough since all channels have the same values.", ) -@click.option( - "--experiment", - "-exp", - help="experiment of ineterst. Word , textline , glyph and textregion are desired options.", -) -def main(dir_xml,dir_out,type_output,experiment,layout_config): - if layout_config: - with open(layout_config) as f: +def main(dir_xml,dir_out,type_output,config): + if config: + with open(config) as f: config_params = json.load(f) else: print("passed") config_params = None - x=pagexml2word(dir_xml,dir_out,type_output,experiment, layout_config) + x=pagexml2label(dir_xml,dir_out,type_output, config) x.run(config_params) if __name__=="__main__": main() From 61487bf782238ff7af96927f2c0c9108191f9ad0 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 23 May 2024 17:36:23 +0200 Subject: [PATCH 053/374] use case printspace is added --- train/pagexml2label.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/train/pagexml2label.py b/train/pagexml2label.py index 16cda8b..94596db 100644 --- a/train/pagexml2label.py +++ b/train/pagexml2label.py @@ -126,7 +126,7 @@ class pagexml2label: y_len=int(jj.attrib['imageHeight']) x_len=int(jj.attrib['imageWidth']) - if self.config and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph'): + if self.config and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph' or config_params['use_case']=='printspace'): keys = list(config_params.keys()) if "artificial_class_label" in keys: artificial_class_rgb_color = (255,255,0) @@ -140,6 +140,9 @@ class pagexml2label: region_tags = np.unique([x for x in alltags if x.endswith('Word')]) elif config_params['use_case']=='glyph': region_tags = np.unique([x for x in alltags if x.endswith('Glyph')]) + elif config_params['use_case']=='printspace': + region_tags = np.unique([x for x in alltags if x.endswith('PrintSpace')]) + co_use_case = [] for tag in region_tags: @@ -149,6 +152,8 @@ class pagexml2label: tag_endings = ['}Word','}word'] elif config_params['use_case']=='glyph': tag_endings = ['}Glyph','}glyph'] + elif config_params['use_case']=='printspace': + tag_endings = ['}PrintSpace','}printspace'] if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): for nn in root1.iter(tag): From d346b317fb5dea9afefa4fd95587f0c8201cd5d7 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 24 May 2024 14:42:58 +0200 Subject: [PATCH 054/374] machine based reading order training dataset generator is added --- train/generate_gt_for_training.py | 194 +++++ train/gt_for_enhancement_creator.py | 31 - train/gt_gen_utils.py | 1239 +++++++++++++++++++++++++++ train/pagexml2label.py | 789 ----------------- 4 files changed, 1433 insertions(+), 820 deletions(-) create mode 100644 train/generate_gt_for_training.py delete mode 100644 train/gt_for_enhancement_creator.py create mode 100644 train/gt_gen_utils.py delete mode 100644 train/pagexml2label.py diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py new file mode 100644 index 0000000..e296029 --- /dev/null +++ b/train/generate_gt_for_training.py @@ -0,0 +1,194 @@ +import click +import json +from gt_gen_utils import * +from tqdm import tqdm + +@click.group() +def main(): + pass + +@main.command() +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--dir_out", + "-do", + help="directory where ground truth images would be written", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--config", + "-cfg", + help="config file of prefered layout or use case.", + type=click.Path(exists=True, dir_okay=False), +) + +@click.option( + "--type_output", + "-to", + help="this defines how output should be. A 2d image array or a 3d image array encoded with RGB color. Just pass 2d or 3d. The file will be saved one directory up. 2D image array is 3d but only information of one channel would be enough since all channels have the same values.", +) + +def pagexml2label(dir_xml,dir_out,type_output,config): + if config: + with open(config) as f: + config_params = json.load(f) + else: + print("passed") + config_params = None + gt_list = get_content_of_dir(dir_xml) + get_images_of_ground_truth(gt_list,dir_xml,dir_out,type_output, config, config_params) + +@main.command() +@click.option( + "--dir_imgs", + "-dis", + help="directory of images with high resolution.", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--dir_out_images", + "-dois", + help="directory where degraded images will be written.", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out_labels", + "-dols", + help="directory where original images will be written as labels.", + type=click.Path(exists=True, file_okay=False), +) +def image_enhancement(dir_imgs, dir_out_images, dir_out_labels): + #dir_imgs = './training_data_sample_enhancement/images' + #dir_out_images = './training_data_sample_enhancement/images_gt' + #dir_out_labels = './training_data_sample_enhancement/labels_gt' + + ls_imgs = os.listdir(dir_imgs) + ls_scales = [ 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] + + for img in tqdm(ls_imgs): + img_name = img.split('.')[0] + img_type = img.split('.')[1] + image = cv2.imread(os.path.join(dir_imgs, img)) + for i, scale in enumerate(ls_scales): + height_sc = int(image.shape[0]*scale) + width_sc = int(image.shape[1]*scale) + + image_down_scaled = resize_image(image, height_sc, width_sc) + image_back_to_org_scale = resize_image(image_down_scaled, image.shape[0], image.shape[1]) + + cv2.imwrite(os.path.join(dir_out_images, img_name+'_'+str(i)+'.'+img_type), image_back_to_org_scale) + cv2.imwrite(os.path.join(dir_out_labels, img_name+'_'+str(i)+'.'+img_type), image) + + +@main.command() +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out_modal_image", + "-domi", + help="directory where ground truth images would be written", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out_classes", + "-docl", + help="directory where ground truth classes would be written", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--input_height", + "-ih", + help="input_height", +) +@click.option( + "--input_width", + "-iw", + help="input_width", +) + +def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width): + xml_files_ind = os.listdir(dir_xml) + input_height = int(input_height) + input_width = int(input_width) + + indexer_start= 0#55166 + max_area = 1 + min_area = 0.0001 + + for ind_xml in tqdm(xml_files_ind): + indexer = 0 + #print(ind_xml) + #print('########################') + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = ind_xml.split('.')[0] + file_name, id_paragraph, id_header,co_text_paragraph,\ + co_text_header,tot_region_ref,x_len, y_len,index_tot_regions,img_poly = read_xml(xml_file) + + id_all_text = id_paragraph + id_header + co_text_all = co_text_paragraph + co_text_header + + + _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours(co_text_header) + + img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') + + for j in range(len(cy_main)): + img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12,int(x_min_main[j]):int(x_max_main[j]) ] = 1 + + + texts_corr_order_index = [index_tot_regions[tot_region_ref.index(i)] for i in id_all_text ] + texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] + + + co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area) + + arg_array = np.array(range(len(texts_corr_order_index_int))) + + labels_con = np.zeros((y_len,x_len,len(arg_array)),dtype='uint8') + for i in range(len(co_text_all)): + img_label = np.zeros((y_len,x_len,3),dtype='uint8') + img_label=cv2.fillPoly(img_label, pts =[co_text_all[i]], color=(1,1,1)) + + img_label[:,:,0][img_poly[:,:,0]==5] = 2 + img_label[:,:,0][img_header_and_sep[:,:]==1] = 3 + + labels_con[:,:,i] = img_label[:,:,0] + + for i in range(len(texts_corr_order_index_int)): + for j in range(len(texts_corr_order_index_int)): + if i!=j: + input_matrix = np.zeros((input_height,input_width,3)).astype(np.int8) + final_f_name = f_name+'_'+str(indexer+indexer_start) + order_class_condition = texts_corr_order_index_int[i]-texts_corr_order_index_int[j] + if order_class_condition<0: + class_type = 1 + else: + class_type = 0 + + input_matrix[:,:,0] = resize_image(labels_con[:,:,i], input_height, input_width) + input_matrix[:,:,1] = resize_image(img_poly[:,:,0], input_height, input_width) + input_matrix[:,:,2] = resize_image(labels_con[:,:,j], input_height, input_width) + + np.save(os.path.join(dir_out_classes,final_f_name+'.npy' ), class_type) + + cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'.png' ), input_matrix) + indexer = indexer+1 + + + +if __name__ == "__main__": + main() diff --git a/train/gt_for_enhancement_creator.py b/train/gt_for_enhancement_creator.py deleted file mode 100644 index 9a4274f..0000000 --- a/train/gt_for_enhancement_creator.py +++ /dev/null @@ -1,31 +0,0 @@ -import cv2 -import os - -def resize_image(seg_in, input_height, input_width): - return cv2.resize(seg_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) - - -dir_imgs = './training_data_sample_enhancement/images' -dir_out_imgs = './training_data_sample_enhancement/images_gt' -dir_out_labs = './training_data_sample_enhancement/labels_gt' - -ls_imgs = os.listdir(dir_imgs) - - -ls_scales = [ 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] - - -for img in ls_imgs: - img_name = img.split('.')[0] - img_type = img.split('.')[1] - image = cv2.imread(os.path.join(dir_imgs, img)) - for i, scale in enumerate(ls_scales): - height_sc = int(image.shape[0]*scale) - width_sc = int(image.shape[1]*scale) - - image_down_scaled = resize_image(image, height_sc, width_sc) - image_back_to_org_scale = resize_image(image_down_scaled, image.shape[0], image.shape[1]) - - cv2.imwrite(os.path.join(dir_out_imgs, img_name+'_'+str(i)+'.'+img_type), image_back_to_org_scale) - cv2.imwrite(os.path.join(dir_out_labs, img_name+'_'+str(i)+'.'+img_type), image) - diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py new file mode 100644 index 0000000..9862e29 --- /dev/null +++ b/train/gt_gen_utils.py @@ -0,0 +1,1239 @@ +import click +import sys +import os +import numpy as np +import warnings +import xml.etree.ElementTree as ET +from tqdm import tqdm +import cv2 +from shapely import geometry +from pathlib import Path + + +KERNEL = np.ones((5, 5), np.uint8) + +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + +def get_content_of_dir(dir_in): + """ + Listing all ground truth page xml files. All files are needed to have xml format. + """ + + gt_all=os.listdir(dir_in) + gt_list=[file for file in gt_all if file.split('.')[ len(file.split('.'))-1 ]=='xml' ] + return gt_list + +def return_parent_contours(contours, hierarchy): + contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1] + return contours_parent +def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, min_area): + found_polygons_early = list() + + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + # area = cv2.contourArea(c) + area = polygon.area + ##print(np.prod(thresh.shape[:2])) + # Check that polygon has area greater than minimal area + # print(hierarchy[0][jv][3],hierarchy ) + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : + # print(c[0][0][1]) + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.int32)) + jv += 1 + return found_polygons_early + +def filter_contours_area_of_image(image, contours, order_index, max_area, min_area): + found_polygons_early = list() + order_index_filtered = list() + #jv = 0 + for jv, c in enumerate(contours): + #print(len(c[0])) + c = c[0] + if len(c) < 3: # A polygon cannot have less than 3 points + continue + c_e = [point for point in c] + #print(c_e) + polygon = geometry.Polygon(c_e) + area = polygon.area + #print(area,'area') + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.uint)) + order_index_filtered.append(order_index[jv]) + #jv += 1 + return found_polygons_early, order_index_filtered + +def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hierarchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) + + return contours_imgs +def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len): + co_text_eroded = [] + for con in co_text: + #try: + img_boundary_in = np.zeros( (y_len,x_len) ) + img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + #print('bidiahhhhaaa') + + + + #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica + if erosion_rate > 0: + img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=erosion_rate) + + pixel = 1 + min_size = 0 + con_eroded = return_contours_of_interested_region(img_boundary_in,pixel, min_size ) + + try: + co_text_eroded.append(con_eroded[0]) + except: + co_text_eroded.append(con) + + + img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_rate) + #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) + + boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] + + img_boundary[:,:][boundary[:,:]==1] =1 + return co_text_eroded, img_boundary +def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_file, config_params): + """ + Reading the page xml files and write the ground truth images into given output directory. + """ + ## to do: add footnote to text regions + for index in tqdm(range(len(gt_list))): + #try: + tree1 = ET.parse(dir_in+'/'+gt_list[index]) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + + if config_file and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph' or config_params['use_case']=='printspace'): + keys = list(config_params.keys()) + if "artificial_class_label" in keys: + artificial_class_rgb_color = (255,255,0) + artificial_class_label = config_params['artificial_class_label'] + + textline_rgb_color = (255, 0, 0) + + if config_params['use_case']=='textline': + region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) + elif config_params['use_case']=='word': + region_tags = np.unique([x for x in alltags if x.endswith('Word')]) + elif config_params['use_case']=='glyph': + region_tags = np.unique([x for x in alltags if x.endswith('Glyph')]) + elif config_params['use_case']=='printspace': + region_tags = np.unique([x for x in alltags if x.endswith('PrintSpace')]) + + co_use_case = [] + + for tag in region_tags: + if config_params['use_case']=='textline': + tag_endings = ['}TextLine','}textline'] + elif config_params['use_case']=='word': + tag_endings = ['}Word','}word'] + elif config_params['use_case']=='glyph': + tag_endings = ['}Glyph','}glyph'] + elif config_params['use_case']=='printspace': + tag_endings = ['}PrintSpace','}printspace'] + + if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + # check the format of coords + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(np.float(vv.attrib['x'])), int(np.float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_use_case.append(np.array(c_t_in)) + + + + if "artificial_class_label" in keys: + img_boundary = np.zeros((y_len, x_len)) + erosion_rate = 1 + dilation_rate = 3 + co_use_case, img_boundary = update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + + + img = np.zeros((y_len, x_len, 3)) + if output_type == '2d': + img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) + if "artificial_class_label" in keys: + img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label + elif output_type == '3d': + img_poly = cv2.fillPoly(img, pts=co_use_case, color=textline_rgb_color) + if "artificial_class_label" in keys: + img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] + img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] + img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + + try: + cv2.imwrite(output_dir + '/' + gt_list[index].split('-')[1].split('.')[0] + '.png', + img_poly) + except: + cv2.imwrite(output_dir + '/' + gt_list[index].split('.')[0] + '.png', img_poly) + + + if config_file and config_params['use_case']=='layout': + keys = list(config_params.keys()) + if "artificial_class_on_boundry" in keys: + elements_with_artificial_class = list(config_params['artificial_class_on_boundry']) + artificial_class_rgb_color = (255,255,0) + artificial_class_label = config_params['artificial_class_label'] + #values = config_params.values() + + if 'textregions' in keys: + types_text_dict = config_params['textregions'] + types_text = list(types_text_dict.keys()) + types_text_label = list(types_text_dict.values()) + print(types_text) + if 'graphicregions' in keys: + types_graphic_dict = config_params['graphicregions'] + types_graphic = list(types_graphic_dict.keys()) + types_graphic_label = list(types_graphic_dict.values()) + + + labels_rgb_color = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (255,0,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,125), (0,255,125), (255,125,255), (125,255,0)] + + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + + co_text_paragraph=[] + co_text_footnote=[] + co_text_footnote_con=[] + co_text_drop=[] + co_text_heading=[] + co_text_header=[] + co_text_marginalia=[] + co_text_catch=[] + co_text_page_number=[] + co_text_signature_mark=[] + co_sep=[] + co_img=[] + co_table=[] + co_graphic_signature=[] + co_graphic_text_annotation=[] + co_graphic_decoration=[] + co_graphic_stamp=[] + co_noise=[] + + for tag in region_tags: + if 'textregions' in keys: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + for nn in root1.iter(tag): + c_t_in_drop=[] + c_t_in_paragraph=[] + c_t_in_heading=[] + c_t_in_header=[] + c_t_in_page_number=[] + c_t_in_signature_mark=[] + c_t_in_catch=[] + c_t_in_marginalia=[] + c_t_in_footnote=[] + c_t_in_footnote_con=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + + coords=bool(vv.attrib) + if coords: + #print('birda1') + p_h=vv.attrib['points'].split(' ') + + if "drop-capital" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "footnote" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote': + c_t_in_footnote.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "footnote-continued" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': + c_t_in_footnote_con.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "heading" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='heading': + c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "signature-mark" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='signature-mark': + c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "header" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='header': + c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "catch-word" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "page-number" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='page-number': + c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "marginalia" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='marginalia': + c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "paragraph" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='paragraph': + c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + break + else: + pass + + + if vv.tag==link+'Point': + if "drop-capital" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "footnote" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote': + c_t_in_footnote.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "footnote-continued" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': + c_t_in_footnote_con.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "heading" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='heading': + c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "signature-mark" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='signature-mark': + c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "header" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='header': + c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "catch-word" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "page-number" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='page-number': + c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "marginalia" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='marginalia': + c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "paragraph" in types_text: + if "type" in nn.attrib and nn.attrib['type']=='paragraph': + c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif vv.tag!=link+'Point' and sumi>=1: + break + + if len(c_t_in_drop)>0: + co_text_drop.append(np.array(c_t_in_drop)) + if len(c_t_in_footnote_con)>0: + co_text_footnote_con.append(np.array(c_t_in_footnote_con)) + if len(c_t_in_footnote)>0: + co_text_footnote.append(np.array(c_t_in_footnote)) + if len(c_t_in_paragraph)>0: + co_text_paragraph.append(np.array(c_t_in_paragraph)) + if len(c_t_in_heading)>0: + co_text_heading.append(np.array(c_t_in_heading)) + + if len(c_t_in_header)>0: + co_text_header.append(np.array(c_t_in_header)) + if len(c_t_in_page_number)>0: + co_text_page_number.append(np.array(c_t_in_page_number)) + if len(c_t_in_catch)>0: + co_text_catch.append(np.array(c_t_in_catch)) + + if len(c_t_in_signature_mark)>0: + co_text_signature_mark.append(np.array(c_t_in_signature_mark)) + + if len(c_t_in_marginalia)>0: + co_text_marginalia.append(np.array(c_t_in_marginalia)) + + + if 'graphicregions' in keys: + if tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in_stamp=[] + c_t_in_text_annotation=[] + c_t_in_decoration=[] + c_t_in_signature=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + if "handwritten-annotation" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "decoration" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='decoration': + c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "stamp" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='stamp': + c_t_in_stamp.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "signature" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='signature': + c_t_in_signature.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + + break + else: + pass + + + if vv.tag==link+'Point': + if "handwritten-annotation" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "decoration" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='decoration': + c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "stamp" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='stamp': + c_t_in_stamp.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if "signature" in types_graphic: + if "type" in nn.attrib and nn.attrib['type']=='signature': + c_t_in_signature.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if len(c_t_in_text_annotation)>0: + co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) + if len(c_t_in_decoration)>0: + co_graphic_decoration.append(np.array(c_t_in_decoration)) + if len(c_t_in_stamp)>0: + co_graphic_stamp.append(np.array(c_t_in_stamp)) + if len(c_t_in_signature)>0: + co_graphic_signature.append(np.array(c_t_in_signature)) + + if 'imageregion' in keys: + if tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + + + if 'separatorregion' in keys: + if tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + + if 'tableregion' in keys: + if tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + + if 'noiseregion' in keys: + if tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_noise.append(np.array(c_t_in)) + + if "artificial_class_on_boundry" in keys: + img_boundary = np.zeros( (y_len,x_len) ) + if "paragraph" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_paragraph, img_boundary = update_region_contours(co_text_paragraph, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "drop-capital" in elements_with_artificial_class: + erosion_rate = 0 + dilation_rate = 4 + co_text_drop, img_boundary = update_region_contours(co_text_drop, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "catch-word" in elements_with_artificial_class: + erosion_rate = 0 + dilation_rate = 4 + co_text_catch, img_boundary = update_region_contours(co_text_catch, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "page-number" in elements_with_artificial_class: + erosion_rate = 0 + dilation_rate = 4 + co_text_page_number, img_boundary = update_region_contours(co_text_page_number, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "header" in elements_with_artificial_class: + erosion_rate = 1 + dilation_rate = 4 + co_text_header, img_boundary = update_region_contours(co_text_header, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "heading" in elements_with_artificial_class: + erosion_rate = 1 + dilation_rate = 4 + co_text_heading, img_boundary = update_region_contours(co_text_heading, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "signature-mark" in elements_with_artificial_class: + erosion_rate = 1 + dilation_rate = 4 + co_text_signature_mark, img_boundary = update_region_contours(co_text_signature_mark, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "marginalia" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_marginalia, img_boundary = update_region_contours(co_text_marginalia, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "footnote" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_footnote, img_boundary = update_region_contours(co_text_footnote, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "footnote-continued" in elements_with_artificial_class: + erosion_rate = 2 + dilation_rate = 4 + co_text_footnote_con, img_boundary = update_region_contours(co_text_footnote_con, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + + + + img = np.zeros( (y_len,x_len,3) ) + + if output_type == '3d': + + if 'graphicregions' in keys: + if "handwritten-annotation" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=labels_rgb_color[ config_params['graphicregions']['handwritten-annotation']]) + if "signature" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=labels_rgb_color[ config_params['graphicregions']['signature']]) + if "decoration" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=labels_rgb_color[ config_params['graphicregions']['decoration']]) + if "stamp" in types_graphic: + img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=labels_rgb_color[ config_params['graphicregions']['stamp']]) + + if 'imageregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_img, color=labels_rgb_color[ config_params['imageregion']]) + if 'separatorregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_sep, color=labels_rgb_color[ config_params['separatorregion']]) + if 'tableregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_table, color=labels_rgb_color[ config_params['tableregion']]) + if 'noiseregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_noise, color=labels_rgb_color[ config_params['noiseregion']]) + + if 'textregions' in keys: + if "paragraph" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=labels_rgb_color[ config_params['textregions']['paragraph']]) + if "footnote" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=labels_rgb_color[ config_params['textregions']['footnote']]) + if "footnote-continued" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=labels_rgb_color[ config_params['textregions']['footnote-continued']]) + if "heading" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=labels_rgb_color[ config_params['textregions']['heading']]) + if "header" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_header, color=labels_rgb_color[ config_params['textregions']['header']]) + if "catch-word" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=labels_rgb_color[ config_params['textregions']['catch-word']]) + if "signature-mark" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=labels_rgb_color[ config_params['textregions']['signature-mark']]) + if "page-number" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=labels_rgb_color[ config_params['textregions']['page-number']]) + if "marginalia" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=labels_rgb_color[ config_params['textregions']['marginalia']]) + if "drop-capital" in types_text: + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=labels_rgb_color[ config_params['textregions']['drop-capital']]) + + if "artificial_class_on_boundry" in keys: + img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] + img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] + img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + + + + + elif output_type == '2d': + if 'graphicregions' in keys: + if "handwritten-annotation" in types_graphic: + color_label = config_params['graphicregions']['handwritten-annotation'] + img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(color_label,color_label,color_label)) + if "signature" in types_graphic: + color_label = config_params['graphicregions']['signature'] + img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=(color_label,color_label,color_label)) + if "decoration" in types_graphic: + color_label = config_params['graphicregions']['decoration'] + img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(color_label,color_label,color_label)) + if "stamp" in types_graphic: + color_label = config_params['graphicregions']['stamp'] + img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=(color_label,color_label,color_label)) + + if 'imageregion' in keys: + color_label = config_params['imageregion'] + img_poly=cv2.fillPoly(img, pts =co_img, color=(color_label,color_label,color_label)) + if 'separatorregion' in keys: + color_label = config_params['separatorregion'] + img_poly=cv2.fillPoly(img, pts =co_sep, color=(color_label,color_label,color_label)) + if 'tableregion' in keys: + color_label = config_params['tableregion'] + img_poly=cv2.fillPoly(img, pts =co_table, color=(color_label,color_label,color_label)) + if 'noiseregion' in keys: + color_label = config_params['noiseregion'] + img_poly=cv2.fillPoly(img, pts =co_noise, color=(color_label,color_label,color_label)) + + if 'textregions' in keys: + if "paragraph" in types_text: + color_label = config_params['textregions']['paragraph'] + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(color_label,color_label,color_label)) + if "footnote" in types_text: + color_label = config_params['textregions']['footnote'] + img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=(color_label,color_label,color_label)) + if "footnote-continued" in types_text: + color_label = config_params['textregions']['footnote-continued'] + img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=(color_label,color_label,color_label)) + if "heading" in types_text: + color_label = config_params['textregions']['heading'] + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(color_label,color_label,color_label)) + if "header" in types_text: + color_label = config_params['textregions']['header'] + img_poly=cv2.fillPoly(img, pts =co_text_header, color=(color_label,color_label,color_label)) + if "catch-word" in types_text: + color_label = config_params['textregions']['catch-word'] + img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(color_label,color_label,color_label)) + if "signature-mark" in types_text: + color_label = config_params['textregions']['signature-mark'] + img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(color_label,color_label,color_label)) + if "page-number" in types_text: + color_label = config_params['textregions']['page-number'] + img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(color_label,color_label,color_label)) + if "marginalia" in types_text: + color_label = config_params['textregions']['marginalia'] + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(color_label,color_label,color_label)) + if "drop-capital" in types_text: + color_label = config_params['textregions']['drop-capital'] + img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(color_label,color_label,color_label)) + + if "artificial_class_on_boundry" in keys: + img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label + + + + + try: + cv2.imwrite(output_dir+'/'+gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + except: + cv2.imwrite(output_dir+'/'+gt_list[index].split('.')[0]+'.png',img_poly ) + + + +def find_new_features_of_contours(contours_main): + + #print(contours_main[0][0][:, 0]) + + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + try: + x_min_main = np.array([np.min(contours_main[j][0][:, 0]) for j in range(len(contours_main))]) + + argmin_x_main = np.array([np.argmin(contours_main[j][0][:, 0]) for j in range(len(contours_main))]) + + x_min_from_argmin = np.array([contours_main[j][0][argmin_x_main[j], 0] for j in range(len(contours_main))]) + y_corr_x_min_from_argmin = np.array([contours_main[j][0][argmin_x_main[j], 1] for j in range(len(contours_main))]) + + x_max_main = np.array([np.max(contours_main[j][0][:, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][0][:, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][0][:, 1]) for j in range(len(contours_main))]) + except: + x_min_main = np.array([np.min(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + argmin_x_main = np.array([np.argmin(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0] for j in range(len(contours_main))]) + y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 1] for j in range(len(contours_main))]) + + x_max_main = np.array([np.max(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 1]) for j in range(len(contours_main))]) + + # dis_x=np.abs(x_max_main-x_min_main) + + return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin +def read_xml(xml_file): + file_name = Path(xml_file).stem + tree1 = ET.parse(xml_file) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + index_tot_regions = [] + tot_region_ref = [] + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + + + for jj in root1.iter(link+'RegionRefIndexed'): + index_tot_regions.append(jj.attrib['index']) + tot_region_ref.append(jj.attrib['regionRef']) + + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + #print(region_tags) + co_text_paragraph=[] + co_text_drop=[] + co_text_heading=[] + co_text_header=[] + co_text_marginalia=[] + co_text_catch=[] + co_text_page_number=[] + co_text_signature_mark=[] + co_sep=[] + co_img=[] + co_table=[] + co_graphic=[] + co_graphic_text_annotation=[] + co_graphic_decoration=[] + co_noise=[] + + + co_text_paragraph_text=[] + co_text_drop_text=[] + co_text_heading_text=[] + co_text_header_text=[] + co_text_marginalia_text=[] + co_text_catch_text=[] + co_text_page_number_text=[] + co_text_signature_mark_text=[] + co_sep_text=[] + co_img_text=[] + co_table_text=[] + co_graphic_text=[] + co_graphic_text_annotation_text=[] + co_graphic_decoration_text=[] + co_noise_text=[] + + + id_paragraph = [] + id_header = [] + id_heading = [] + id_marginalia = [] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + for nn in root1.iter(tag): + for child2 in nn: + tag2 = child2.tag + #print(child2.tag) + if tag2.endswith('}TextEquiv') or tag2.endswith('}TextEquiv'): + #children2 = childtext.getchildren() + #rank = child2.find('Unicode').text + for childtext2 in child2: + #rank = childtext2.find('Unicode').text + #if childtext2.tag.endswith('}PlainText') or childtext2.tag.endswith('}PlainText'): + #print(childtext2.text) + if childtext2.tag.endswith('}Unicode') or childtext2.tag.endswith('}Unicode'): + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + co_text_drop_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='heading': + co_text_heading_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + co_text_signature_mark_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='header': + co_text_header_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + co_text_catch_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='page-number': + co_text_page_number_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + co_text_marginalia_text.append(childtext2.text) + else: + co_text_paragraph_text.append(childtext2.text) + c_t_in_drop=[] + c_t_in_paragraph=[] + c_t_in_heading=[] + c_t_in_header=[] + c_t_in_page_number=[] + c_t_in_signature_mark=[] + c_t_in_catch=[] + c_t_in_marginalia=[] + + + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + + coords=bool(vv.attrib) + if coords: + #print('birda1') + p_h=vv.attrib['points'].split(' ') + + + + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + #if nn.attrib['type']=='paragraph': + + c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='heading': + id_heading.append(nn.attrib['id']) + c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + + c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + elif "type" in nn.attrib and nn.attrib['type']=='header': + id_header.append(nn.attrib['id']) + c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='page-number': + + c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + id_marginalia.append(nn.attrib['id']) + + c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + else: + #print(nn.attrib['id']) + + id_paragraph.append(nn.attrib['id']) + + c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + + break + else: + pass + + + if vv.tag==link+'Point': + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + #if nn.attrib['type']=='paragraph': + + c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='heading': + id_heading.append(nn.attrib['id']) + c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + + c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + elif "type" in nn.attrib and nn.attrib['type']=='header': + id_header.append(nn.attrib['id']) + c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='page-number': + + c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + id_marginalia.append(nn.attrib['id']) + + c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + + else: + id_paragraph.append(nn.attrib['id']) + c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + + #c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + + if len(c_t_in_drop)>0: + co_text_drop.append(np.array(c_t_in_drop)) + if len(c_t_in_paragraph)>0: + co_text_paragraph.append(np.array(c_t_in_paragraph)) + if len(c_t_in_heading)>0: + co_text_heading.append(np.array(c_t_in_heading)) + + if len(c_t_in_header)>0: + co_text_header.append(np.array(c_t_in_header)) + if len(c_t_in_page_number)>0: + co_text_page_number.append(np.array(c_t_in_page_number)) + if len(c_t_in_catch)>0: + co_text_catch.append(np.array(c_t_in_catch)) + + if len(c_t_in_signature_mark)>0: + co_text_signature_mark.append(np.array(c_t_in_signature_mark)) + + if len(c_t_in_marginalia)>0: + co_text_marginalia.append(np.array(c_t_in_marginalia)) + + + elif tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + c_t_in_text_annotation=[] + c_t_in_decoration=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + #c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + #if nn.attrib['type']=='paragraph': + + c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='decoration': + + c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + else: + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + + break + else: + pass + + + if vv.tag==link+'Point': + + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + #if nn.attrib['type']=='paragraph': + + c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='decoration': + + c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #print(c_t_in_paragraph) + sumi+=1 + else: + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + + if len(c_t_in_text_annotation)>0: + co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) + if len(c_t_in_decoration)>0: + co_graphic_decoration.append(np.array(c_t_in_decoration)) + if len(c_t_in)>0: + co_graphic.append(np.array(c_t_in)) + + + + elif tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + co_img_text.append(' ') + + + elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + + elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + co_table_text.append(' ') + + elif tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_noise.append(np.array(c_t_in)) + co_noise_text.append(' ') + + + img = np.zeros( (y_len,x_len,3) ) + + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(1,1,1)) + + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(2,2,2)) + img_poly=cv2.fillPoly(img, pts =co_text_header, color=(2,2,2)) + #img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(125,255,125)) + #img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(125,125,0)) + #img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(1,125,255)) + #img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(1,125,0)) + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(3,3,3)) + #img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(1,125,255)) + + #img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(125,0,125)) + img_poly=cv2.fillPoly(img, pts =co_img, color=(4,4,4)) + img_poly=cv2.fillPoly(img, pts =co_sep, color=(5,5,5)) + #img_poly=cv2.fillPoly(img, pts =co_table, color=(1,255,255)) + #img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) + #img_poly=cv2.fillPoly(img, pts =co_noise, color=(255,0,255)) + + #print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.jpg') + ###try: + ####print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.jpg') + ###cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.jpg',img_poly ) + ###except: + ###cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.jpg',img_poly ) + return file_name, id_paragraph, id_header,co_text_paragraph, co_text_header,\ +tot_region_ref,x_len, y_len,index_tot_regions, img_poly + + + + +def bounding_box(cnt,color, corr_order_index ): + x, y, w, h = cv2.boundingRect(cnt) + x = int(x*scale_w) + y = int(y*scale_h) + + w = int(w*scale_w) + h = int(h*scale_h) + + return [x,y,w,h,int(color), int(corr_order_index)+1] + +def resize_image(seg_in,input_height,input_width): + return cv2.resize(seg_in,(input_width,input_height),interpolation=cv2.INTER_NEAREST) + +def make_image_from_bb(width_l, height_l, bb_all): + bb_all =np.array(bb_all) + img_remade = np.zeros((height_l,width_l )) + + for i in range(bb_all.shape[0]): + img_remade[bb_all[i,1]:bb_all[i,1]+bb_all[i,3],bb_all[i,0]:bb_all[i,0]+bb_all[i,2] ] = 1 + return img_remade diff --git a/train/pagexml2label.py b/train/pagexml2label.py deleted file mode 100644 index 94596db..0000000 --- a/train/pagexml2label.py +++ /dev/null @@ -1,789 +0,0 @@ -import click -import sys -import os -import numpy as np -import warnings -import xml.etree.ElementTree as ET -from tqdm import tqdm -import cv2 -from shapely import geometry -import json - -with warnings.catch_warnings(): - warnings.simplefilter("ignore") - -__doc__=\ -""" -tool to extract 2d or 3d RGB images from page xml data. In former case output will be 1 -2D image array which each class has filled with a pixel value. In the case of 3D RGB image -each class will be defined with a RGB value and beside images a text file of classes also will be produced. -This classes.txt file is required for dhsegment tool. -""" -KERNEL = np.ones((5, 5), np.uint8) - -class pagexml2label: - def __init__(self,dir_in, out_dir,output_type,config): - self.dir=dir_in - self.output_dir=out_dir - self.output_type=output_type - self.config=config - - def get_content_of_dir(self): - """ - Listing all ground truth page xml files. All files are needed to have xml format. - """ - - gt_all=os.listdir(self.dir) - self.gt_list=[file for file in gt_all if file.split('.')[ len(file.split('.'))-1 ]=='xml' ] - - def return_parent_contours(self,contours, hierarchy): - contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1] - return contours_parent - def filter_contours_area_of_image_tables(self,image, contours, hierarchy, max_area, min_area): - found_polygons_early = list() - - jv = 0 - for c in contours: - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - # area = cv2.contourArea(c) - area = polygon.area - ##print(np.prod(thresh.shape[:2])) - # Check that polygon has area greater than minimal area - # print(hierarchy[0][jv][3],hierarchy ) - if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : - # print(c[0][0][1]) - found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.int32)) - jv += 1 - return found_polygons_early - - def return_contours_of_interested_region(self,region_pre_p, pixel, min_area=0.0002): - - # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 - else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs = self.return_parent_contours(contours_imgs, hierarchy) - contours_imgs = self.filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) - - return contours_imgs - def update_region_contours(self, co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len): - co_text_eroded = [] - for con in co_text: - #try: - img_boundary_in = np.zeros( (y_len,x_len) ) - img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - #print('bidiahhhhaaa') - - - - #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica - if erosion_rate > 0: - img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=erosion_rate) - - pixel = 1 - min_size = 0 - con_eroded = self.return_contours_of_interested_region(img_boundary_in,pixel, min_size ) - - try: - co_text_eroded.append(con_eroded[0]) - except: - co_text_eroded.append(con) - - - img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_rate) - #img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=5) - - boundary = img_boundary_in_dilated[:,:] - img_boundary_in[:,:] - - img_boundary[:,:][boundary[:,:]==1] =1 - return co_text_eroded, img_boundary - def get_images_of_ground_truth(self, config_params): - """ - Reading the page xml files and write the ground truth images into given output directory. - """ - ## to do: add footnote to text regions - for index in tqdm(range(len(self.gt_list))): - #try: - tree1 = ET.parse(self.dir+'/'+self.gt_list[index]) - root1=tree1.getroot() - alltags=[elem.tag for elem in root1.iter()] - link=alltags[0].split('}')[0]+'}' - - - - for jj in root1.iter(link+'Page'): - y_len=int(jj.attrib['imageHeight']) - x_len=int(jj.attrib['imageWidth']) - - if self.config and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph' or config_params['use_case']=='printspace'): - keys = list(config_params.keys()) - if "artificial_class_label" in keys: - artificial_class_rgb_color = (255,255,0) - artificial_class_label = config_params['artificial_class_label'] - - textline_rgb_color = (255, 0, 0) - - if config_params['use_case']=='textline': - region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) - elif config_params['use_case']=='word': - region_tags = np.unique([x for x in alltags if x.endswith('Word')]) - elif config_params['use_case']=='glyph': - region_tags = np.unique([x for x in alltags if x.endswith('Glyph')]) - elif config_params['use_case']=='printspace': - region_tags = np.unique([x for x in alltags if x.endswith('PrintSpace')]) - - co_use_case = [] - - for tag in region_tags: - if config_params['use_case']=='textline': - tag_endings = ['}TextLine','}textline'] - elif config_params['use_case']=='word': - tag_endings = ['}Word','}word'] - elif config_params['use_case']=='glyph': - tag_endings = ['}Glyph','}glyph'] - elif config_params['use_case']=='printspace': - tag_endings = ['}PrintSpace','}printspace'] - - if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): - for nn in root1.iter(tag): - c_t_in = [] - sumi = 0 - for vv in nn.iter(): - # check the format of coords - if vv.tag == link + 'Coords': - coords = bool(vv.attrib) - if coords: - p_h = vv.attrib['points'].split(' ') - c_t_in.append( - np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) - break - else: - pass - - if vv.tag == link + 'Point': - c_t_in.append([int(np.float(vv.attrib['x'])), int(np.float(vv.attrib['y']))]) - sumi += 1 - elif vv.tag != link + 'Point' and sumi >= 1: - break - co_use_case.append(np.array(c_t_in)) - - - - if "artificial_class_label" in keys: - img_boundary = np.zeros((y_len, x_len)) - erosion_rate = 1 - dilation_rate = 3 - co_use_case, img_boundary = self.update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - - - img = np.zeros((y_len, x_len, 3)) - if self.output_type == '2d': - img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) - if "artificial_class_label" in keys: - img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label - elif self.output_type == '3d': - img_poly = cv2.fillPoly(img, pts=co_use_case, color=textline_rgb_color) - if "artificial_class_label" in keys: - img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] - img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] - img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] - - try: - cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('-')[1].split('.')[0] + '.png', - img_poly) - except: - cv2.imwrite(self.output_dir + '/' + self.gt_list[index].split('.')[0] + '.png', img_poly) - - - if self.config and config_params['use_case']=='layout': - keys = list(config_params.keys()) - if "artificial_class_on_boundry" in keys: - elements_with_artificial_class = list(config_params['artificial_class_on_boundry']) - artificial_class_rgb_color = (255,255,0) - artificial_class_label = config_params['artificial_class_label'] - #values = config_params.values() - - if 'textregions' in keys: - types_text_dict = config_params['textregions'] - types_text = list(types_text_dict.keys()) - types_text_label = list(types_text_dict.values()) - print(types_text) - if 'graphicregions' in keys: - types_graphic_dict = config_params['graphicregions'] - types_graphic = list(types_graphic_dict.keys()) - types_graphic_label = list(types_graphic_dict.values()) - - - labels_rgb_color = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (255,0,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,125), (0,255,125), (255,125,255), (125,255,0)] - - region_tags=np.unique([x for x in alltags if x.endswith('Region')]) - - co_text_paragraph=[] - co_text_footnote=[] - co_text_footnote_con=[] - co_text_drop=[] - co_text_heading=[] - co_text_header=[] - co_text_marginalia=[] - co_text_catch=[] - co_text_page_number=[] - co_text_signature_mark=[] - co_sep=[] - co_img=[] - co_table=[] - co_graphic_signature=[] - co_graphic_text_annotation=[] - co_graphic_decoration=[] - co_graphic_stamp=[] - co_noise=[] - - for tag in region_tags: - if 'textregions' in keys: - if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): - for nn in root1.iter(tag): - c_t_in_drop=[] - c_t_in_paragraph=[] - c_t_in_heading=[] - c_t_in_header=[] - c_t_in_page_number=[] - c_t_in_signature_mark=[] - c_t_in_catch=[] - c_t_in_marginalia=[] - c_t_in_footnote=[] - c_t_in_footnote_con=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - - coords=bool(vv.attrib) - if coords: - #print('birda1') - p_h=vv.attrib['points'].split(' ') - - if "drop-capital" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "footnote" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote': - c_t_in_footnote.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "footnote-continued" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': - c_t_in_footnote_con.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "heading" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='heading': - c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "signature-mark" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='signature-mark': - c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "header" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='header': - c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "catch-word" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "page-number" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='page-number': - c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "marginalia" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='marginalia': - c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "paragraph" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='paragraph': - c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - - break - else: - pass - - - if vv.tag==link+'Point': - if "drop-capital" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "footnote" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote': - c_t_in_footnote.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "footnote-continued" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': - c_t_in_footnote_con.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "heading" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='heading': - c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "signature-mark" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='signature-mark': - c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "header" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='header': - c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "catch-word" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "page-number" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='page-number': - c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "marginalia" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='marginalia': - c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "paragraph" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='paragraph': - c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - - elif vv.tag!=link+'Point' and sumi>=1: - break - - if len(c_t_in_drop)>0: - co_text_drop.append(np.array(c_t_in_drop)) - if len(c_t_in_footnote_con)>0: - co_text_footnote_con.append(np.array(c_t_in_footnote_con)) - if len(c_t_in_footnote)>0: - co_text_footnote.append(np.array(c_t_in_footnote)) - if len(c_t_in_paragraph)>0: - co_text_paragraph.append(np.array(c_t_in_paragraph)) - if len(c_t_in_heading)>0: - co_text_heading.append(np.array(c_t_in_heading)) - - if len(c_t_in_header)>0: - co_text_header.append(np.array(c_t_in_header)) - if len(c_t_in_page_number)>0: - co_text_page_number.append(np.array(c_t_in_page_number)) - if len(c_t_in_catch)>0: - co_text_catch.append(np.array(c_t_in_catch)) - - if len(c_t_in_signature_mark)>0: - co_text_signature_mark.append(np.array(c_t_in_signature_mark)) - - if len(c_t_in_marginalia)>0: - co_text_marginalia.append(np.array(c_t_in_marginalia)) - - - if 'graphicregions' in keys: - if tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in_stamp=[] - c_t_in_text_annotation=[] - c_t_in_decoration=[] - c_t_in_signature=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - if "handwritten-annotation" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "decoration" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "stamp" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='stamp': - c_t_in_stamp.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "signature" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='signature': - c_t_in_signature.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - - - break - else: - pass - - - if vv.tag==link+'Point': - if "handwritten-annotation" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "decoration" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "stamp" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='stamp': - c_t_in_stamp.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "signature" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='signature': - c_t_in_signature.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if len(c_t_in_text_annotation)>0: - co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) - if len(c_t_in_decoration)>0: - co_graphic_decoration.append(np.array(c_t_in_decoration)) - if len(c_t_in_stamp)>0: - co_graphic_stamp.append(np.array(c_t_in_stamp)) - if len(c_t_in_signature)>0: - co_graphic_signature.append(np.array(c_t_in_signature)) - - if 'imageregion' in keys: - if tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - elif vv.tag!=link+'Point' and sumi>=1: - break - co_img.append(np.array(c_t_in)) - - - if 'separatorregion' in keys: - if tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - elif vv.tag!=link+'Point' and sumi>=1: - break - co_sep.append(np.array(c_t_in)) - - - - if 'tableregion' in keys: - if tag.endswith('}TableRegion') or tag.endswith('}tableregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_table.append(np.array(c_t_in)) - - if 'noiseregion' in keys: - if tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): - #print('sth') - for nn in root1.iter(tag): - c_t_in=[] - sumi=0 - for vv in nn.iter(): - # check the format of coords - if vv.tag==link+'Coords': - coords=bool(vv.attrib) - if coords: - p_h=vv.attrib['points'].split(' ') - c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break - else: - pass - - - if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - #print(vv.tag,'in') - elif vv.tag!=link+'Point' and sumi>=1: - break - co_noise.append(np.array(c_t_in)) - - if "artificial_class_on_boundry" in keys: - img_boundary = np.zeros( (y_len,x_len) ) - if "paragraph" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 - co_text_paragraph, img_boundary = self.update_region_contours(co_text_paragraph, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "drop-capital" in elements_with_artificial_class: - erosion_rate = 0 - dilation_rate = 4 - co_text_drop, img_boundary = self.update_region_contours(co_text_drop, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "catch-word" in elements_with_artificial_class: - erosion_rate = 0 - dilation_rate = 4 - co_text_catch, img_boundary = self.update_region_contours(co_text_catch, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "page-number" in elements_with_artificial_class: - erosion_rate = 0 - dilation_rate = 4 - co_text_page_number, img_boundary = self.update_region_contours(co_text_page_number, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "header" in elements_with_artificial_class: - erosion_rate = 1 - dilation_rate = 4 - co_text_header, img_boundary = self.update_region_contours(co_text_header, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "heading" in elements_with_artificial_class: - erosion_rate = 1 - dilation_rate = 4 - co_text_heading, img_boundary = self.update_region_contours(co_text_heading, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "signature-mark" in elements_with_artificial_class: - erosion_rate = 1 - dilation_rate = 4 - co_text_signature_mark, img_boundary = self.update_region_contours(co_text_signature_mark, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "marginalia" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 - co_text_marginalia, img_boundary = self.update_region_contours(co_text_marginalia, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "footnote" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 - co_text_footnote, img_boundary = self.update_region_contours(co_text_footnote, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - if "footnote-continued" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 - co_text_footnote_con, img_boundary = self.update_region_contours(co_text_footnote_con, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) - - - - img = np.zeros( (y_len,x_len,3) ) - - if self.output_type == '3d': - - if 'graphicregions' in keys: - if "handwritten-annotation" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=labels_rgb_color[ config_params['graphicregions']['handwritten-annotation']]) - if "signature" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=labels_rgb_color[ config_params['graphicregions']['signature']]) - if "decoration" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=labels_rgb_color[ config_params['graphicregions']['decoration']]) - if "stamp" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=labels_rgb_color[ config_params['graphicregions']['stamp']]) - - if 'imageregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_img, color=labels_rgb_color[ config_params['imageregion']]) - if 'separatorregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_sep, color=labels_rgb_color[ config_params['separatorregion']]) - if 'tableregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_table, color=labels_rgb_color[ config_params['tableregion']]) - if 'noiseregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_noise, color=labels_rgb_color[ config_params['noiseregion']]) - - if 'textregions' in keys: - if "paragraph" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=labels_rgb_color[ config_params['textregions']['paragraph']]) - if "footnote" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=labels_rgb_color[ config_params['textregions']['footnote']]) - if "footnote-continued" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=labels_rgb_color[ config_params['textregions']['footnote-continued']]) - if "heading" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=labels_rgb_color[ config_params['textregions']['heading']]) - if "header" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_header, color=labels_rgb_color[ config_params['textregions']['header']]) - if "catch-word" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=labels_rgb_color[ config_params['textregions']['catch-word']]) - if "signature-mark" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=labels_rgb_color[ config_params['textregions']['signature-mark']]) - if "page-number" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=labels_rgb_color[ config_params['textregions']['page-number']]) - if "marginalia" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=labels_rgb_color[ config_params['textregions']['marginalia']]) - if "drop-capital" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=labels_rgb_color[ config_params['textregions']['drop-capital']]) - - if "artificial_class_on_boundry" in keys: - img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] - img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] - img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] - - - - - elif self.output_type == '2d': - if 'graphicregions' in keys: - if "handwritten-annotation" in types_graphic: - color_label = config_params['graphicregions']['handwritten-annotation'] - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(color_label,color_label,color_label)) - if "signature" in types_graphic: - color_label = config_params['graphicregions']['signature'] - img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=(color_label,color_label,color_label)) - if "decoration" in types_graphic: - color_label = config_params['graphicregions']['decoration'] - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(color_label,color_label,color_label)) - if "stamp" in types_graphic: - color_label = config_params['graphicregions']['stamp'] - img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=(color_label,color_label,color_label)) - - if 'imageregion' in keys: - color_label = config_params['imageregion'] - img_poly=cv2.fillPoly(img, pts =co_img, color=(color_label,color_label,color_label)) - if 'separatorregion' in keys: - color_label = config_params['separatorregion'] - img_poly=cv2.fillPoly(img, pts =co_sep, color=(color_label,color_label,color_label)) - if 'tableregion' in keys: - color_label = config_params['tableregion'] - img_poly=cv2.fillPoly(img, pts =co_table, color=(color_label,color_label,color_label)) - if 'noiseregion' in keys: - color_label = config_params['noiseregion'] - img_poly=cv2.fillPoly(img, pts =co_noise, color=(color_label,color_label,color_label)) - - if 'textregions' in keys: - if "paragraph" in types_text: - color_label = config_params['textregions']['paragraph'] - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(color_label,color_label,color_label)) - if "footnote" in types_text: - color_label = config_params['textregions']['footnote'] - img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=(color_label,color_label,color_label)) - if "footnote-continued" in types_text: - color_label = config_params['textregions']['footnote-continued'] - img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=(color_label,color_label,color_label)) - if "heading" in types_text: - color_label = config_params['textregions']['heading'] - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(color_label,color_label,color_label)) - if "header" in types_text: - color_label = config_params['textregions']['header'] - img_poly=cv2.fillPoly(img, pts =co_text_header, color=(color_label,color_label,color_label)) - if "catch-word" in types_text: - color_label = config_params['textregions']['catch-word'] - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(color_label,color_label,color_label)) - if "signature-mark" in types_text: - color_label = config_params['textregions']['signature-mark'] - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(color_label,color_label,color_label)) - if "page-number" in types_text: - color_label = config_params['textregions']['page-number'] - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(color_label,color_label,color_label)) - if "marginalia" in types_text: - color_label = config_params['textregions']['marginalia'] - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(color_label,color_label,color_label)) - if "drop-capital" in types_text: - color_label = config_params['textregions']['drop-capital'] - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(color_label,color_label,color_label)) - - if "artificial_class_on_boundry" in keys: - img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label - - - - - try: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) - except: - cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.png',img_poly ) - - - def run(self,config_params): - self.get_content_of_dir() - self.get_images_of_ground_truth(config_params) - - -@click.command() -@click.option( - "--dir_xml", - "-dx", - help="directory of GT page-xml files", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--dir_out", - "-do", - help="directory where ground truth images would be written", - type=click.Path(exists=True, file_okay=False), -) - -@click.option( - "--config", - "-cfg", - help="config file of prefered layout or use case.", - type=click.Path(exists=True, dir_okay=False), -) - -@click.option( - "--type_output", - "-to", - help="this defines how output should be. A 2d image array or a 3d image array encoded with RGB color. Just pass 2d or 3d. The file will be saved one directory up. 2D image array is 3d but only information of one channel would be enough since all channels have the same values.", -) - - -def main(dir_xml,dir_out,type_output,config): - if config: - with open(config) as f: - config_params = json.load(f) - else: - print("passed") - config_params = None - x=pagexml2label(dir_xml,dir_out,type_output, config) - x.run(config_params) -if __name__=="__main__": - main() - - - From 9638098ae7e5269a597a98937f3c239270575525 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 24 May 2024 16:39:48 +0200 Subject: [PATCH 055/374] machine based reading order training is integrated --- train/models.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ train/train.py | 31 ++++++++++++++++++++++++++++ train/utils.py | 23 +++++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/train/models.py b/train/models.py index 4cceacd..d852ac3 100644 --- a/train/models.py +++ b/train/models.py @@ -544,4 +544,59 @@ def resnet50_classifier(n_classes,input_height=224,input_width=224,weight_decay= + return model + +def machine_based_reading_order_model(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): + assert input_height%32 == 0 + assert input_width%32 == 0 + + img_input = Input(shape=(input_height,input_width , 3 )) + + if IMAGE_ORDERING == 'channels_last': + bn_axis = 3 + else: + bn_axis = 1 + + x1 = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(img_input) + x1 = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x1) + + x1 = BatchNormalization(axis=bn_axis, name='bn_conv1')(x1) + x1 = Activation('relu')(x1) + x1 = MaxPooling2D((3, 3) , data_format=IMAGE_ORDERING , strides=(2, 2))(x1) + + x1 = conv_block(x1, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x1 = identity_block(x1, 3, [64, 64, 256], stage=2, block='b') + x1 = identity_block(x1, 3, [64, 64, 256], stage=2, block='c') + + x1 = conv_block(x1, 3, [128, 128, 512], stage=3, block='a') + x1 = identity_block(x1, 3, [128, 128, 512], stage=3, block='b') + x1 = identity_block(x1, 3, [128, 128, 512], stage=3, block='c') + x1 = identity_block(x1, 3, [128, 128, 512], stage=3, block='d') + + x1 = conv_block(x1, 3, [256, 256, 1024], stage=4, block='a') + x1 = identity_block(x1, 3, [256, 256, 1024], stage=4, block='b') + x1 = identity_block(x1, 3, [256, 256, 1024], stage=4, block='c') + x1 = identity_block(x1, 3, [256, 256, 1024], stage=4, block='d') + x1 = identity_block(x1, 3, [256, 256, 1024], stage=4, block='e') + x1 = identity_block(x1, 3, [256, 256, 1024], stage=4, block='f') + + x1 = conv_block(x1, 3, [512, 512, 2048], stage=5, block='a') + x1 = identity_block(x1, 3, [512, 512, 2048], stage=5, block='b') + x1 = identity_block(x1, 3, [512, 512, 2048], stage=5, block='c') + + if pretraining: + Model(img_input , x1).load_weights(resnet50_Weights_path) + + x1 = AveragePooling2D((7, 7), name='avg_pool1')(x1) + flattened = Flatten()(x1) + + o = Dense(256, activation='relu', name='fc512')(flattened) + o=Dropout(0.2)(o) + + o = Dense(256, activation='relu', name='fc512a')(o) + o=Dropout(0.2)(o) + + o = Dense(n_classes, activation='sigmoid', name='fc1000')(o) + model = Model(img_input , o) + return model diff --git a/train/train.py b/train/train.py index 78974d3..f338c78 100644 --- a/train/train.py +++ b/train/train.py @@ -313,4 +313,35 @@ def run(_config, n_classes, n_epochs, input_height, with open(os.path.join( os.path.join(dir_output,'model_best'), "config.json"), "w") as fp: json.dump(_config, fp) # encode dict into JSON + + elif task=='reading_order': + configuration() + model = machine_based_reading_order_model(n_classes,input_height,input_width,weight_decay,pretraining) + + dir_flow_train_imgs = os.path.join(dir_train, 'images') + dir_flow_train_labels = os.path.join(dir_train, 'labels') + + classes = os.listdir(dir_flow_train_labels) + num_rows =len(classes) + #ls_test = os.listdir(dir_flow_train_labels) + + #f1score_tot = [0] + indexer_start = 0 + opt = SGD(lr=0.01, momentum=0.9) + opt_adam = tf.keras.optimizers.Adam(learning_rate=0.0001) + model.compile(loss="binary_crossentropy", + optimizer = opt_adam,metrics=['accuracy']) + for i in range(n_epochs): + history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes), steps_per_epoch=num_rows / n_batch, verbose=1) + model.save( os.path.join(dir_output,'model_'+str(i+indexer_start) )) + + with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: + json.dump(_config, fp) # encode dict into JSON + ''' + if f1score>f1score_tot[0]: + f1score_tot[0] = f1score + model_dir = os.path.join(dir_out,'model_best') + model.save(model_dir) + ''' + diff --git a/train/utils.py b/train/utils.py index 271d977..a2e8a9c 100644 --- a/train/utils.py +++ b/train/utils.py @@ -268,6 +268,29 @@ def IoU(Yi, y_predi): #print("Mean IoU: {:4.3f}".format(mIoU)) return mIoU +def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batchsize, height, width, n_classes): + all_labels_files = os.listdir(classes_file_dir) + ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) + ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + batchcount = 0 + while True: + for i in all_labels_files: + file_name = i.split('.')[0] + img = cv2.imread(os.path.join(modal_dir,file_name+'.png')) + + label_class = int( np.load(os.path.join(classes_file_dir,i)) ) + + ret_x[batchcount, :,:,0] = img[:,:,0]/3.0 + ret_x[batchcount, :,:,2] = img[:,:,2]/3.0 + ret_x[batchcount, :,:,1] = img[:,:,1]/5.0 + + ret_y[batchcount, :] = label_class + batchcount+=1 + if batchcount>=batchsize: + yield (ret_x, ret_y) + ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) + ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + batchcount = 0 def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_classes, task='segmentation'): c = 0 From ccf520d3c73d7c1132509434a206ddb2d504b5c2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 27 May 2024 17:23:49 +0200 Subject: [PATCH 056/374] adding rest_as_paragraph and rest_as_graphic to elements --- train/custom_config_page2label.json | 10 +- train/gt_gen_utils.py | 454 ++++++++++------------------ 2 files changed, 170 insertions(+), 294 deletions(-) diff --git a/train/custom_config_page2label.json b/train/custom_config_page2label.json index d6320fa..e4c02cb 100644 --- a/train/custom_config_page2label.json +++ b/train/custom_config_page2label.json @@ -1,9 +1,9 @@ { "use_case": "layout", -"textregions":{"paragraph":1, "heading": 2, "header":2,"drop-capital": 3, "marginalia":4 ,"page-number":1 , "catch-word":1 ,"footnote": 1, "footnote-continued": 1}, -"imageregion":5, -"separatorregion":6, -"graphicregions" :{"handwritten-annotation":5, "decoration": 5, "signature": 5, "stamp": 5}, -"artificial_class_on_boundry": ["paragraph","header", "heading", "marginalia", "page-number", "catch-word", "drop-capital","footnote", "footnote-continued"], +"textregions":{ "rest_as_paragraph": 1, "header":2 , "heading":2 , "marginalia":3 }, +"imageregion":4, +"separatorregion":5, +"graphicregions" :{"rest_as_decoration":6}, +"artificial_class_on_boundry": ["paragraph"], "artificial_class_label":7 } diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 9862e29..9dc8377 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -180,7 +180,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ pass if vv.tag == link + 'Point': - c_t_in.append([int(np.float(vv.attrib['x'])), int(np.float(vv.attrib['y']))]) + c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) sumi += 1 elif vv.tag != link + 'Point' and sumi >= 1: break @@ -226,7 +226,6 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ types_text_dict = config_params['textregions'] types_text = list(types_text_dict.keys()) types_text_label = list(types_text_dict.values()) - print(types_text) if 'graphicregions' in keys: types_graphic_dict = config_params['graphicregions'] types_graphic = list(types_graphic_dict.keys()) @@ -235,41 +234,20 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ labels_rgb_color = [ (0,0,0), (255,0,0), (255,125,0), (255,0,125), (125,255,125), (125,125,0), (0,125,255), (0,125,0), (125,125,125), (255,0,255), (125,0,125), (0,255,0),(0,0,255), (0,255,255), (255,125,125), (0,125,125), (0,255,125), (255,125,255), (125,255,0)] + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) - - co_text_paragraph=[] - co_text_footnote=[] - co_text_footnote_con=[] - co_text_drop=[] - co_text_heading=[] - co_text_header=[] - co_text_marginalia=[] - co_text_catch=[] - co_text_page_number=[] - co_text_signature_mark=[] + co_text = {'drop-capital':[], "footnote":[], "footnote-continued":[], "heading":[], "signature-mark":[], "header":[], "catch-word":[], "page-number":[], "marginalia":[], "paragraph":[]} + co_graphic = {"handwritten-annotation":[], "decoration":[], "stamp":[], "signature":[]} co_sep=[] co_img=[] co_table=[] - co_graphic_signature=[] - co_graphic_text_annotation=[] - co_graphic_decoration=[] - co_graphic_stamp=[] co_noise=[] for tag in region_tags: if 'textregions' in keys: if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): for nn in root1.iter(tag): - c_t_in_drop=[] - c_t_in_paragraph=[] - c_t_in_heading=[] - c_t_in_header=[] - c_t_in_page_number=[] - c_t_in_signature_mark=[] - c_t_in_catch=[] - c_t_in_marginalia=[] - c_t_in_footnote=[] - c_t_in_footnote_con=[] + c_t_in = {'drop-capital':[], "footnote":[], "footnote-continued":[], "heading":[], "signature-mark":[], "header":[], "catch-word":[], "page-number":[], "marginalia":[], "paragraph":[]} sumi=0 for vv in nn.iter(): # check the format of coords @@ -277,143 +255,63 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ coords=bool(vv.attrib) if coords: - #print('birda1') p_h=vv.attrib['points'].split(' ') - if "drop-capital" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "footnote" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote': - c_t_in_footnote.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "footnote-continued" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': - c_t_in_footnote_con.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "heading" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='heading': - c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "signature-mark" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='signature-mark': - c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "header" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='header': - c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "catch-word" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "page-number" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='page-number': - c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "marginalia" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='marginalia': - c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "paragraph" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='paragraph': - c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - + if "rest_as_paragraph" in types_text: + types_text_without_paragraph = [element for element in types_text if element!='rest_as_paragraph' and element!='paragraph'] + if len(types_text_without_paragraph) == 0: + if "type" in nn.attrib: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + elif len(types_text_without_paragraph) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_text_without_paragraph: + c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + else: + if "type" in nn.attrib: + c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) break else: pass - + if vv.tag==link+'Point': - if "drop-capital" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + if "rest_as_paragraph" in types_text: + types_text_without_paragraph = [element for element in types_text if element!='rest_as_paragraph' and element!='paragraph'] + if len(types_text_without_paragraph) == 0: + if "type" in nn.attrib: + c_t_in['paragraph'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + elif len(types_text_without_paragraph) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_text_without_paragraph: + c_t_in[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + else: + c_t_in['paragraph'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + + else: + if "type" in nn.attrib: + c_t_in[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) sumi+=1 - - if "footnote" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote': - c_t_in_footnote.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "footnote-continued" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='footnote-continued': - c_t_in_footnote_con.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "heading" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='heading': - c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "signature-mark" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='signature-mark': - c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "header" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='header': - c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "catch-word" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "page-number" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='page-number': - c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "marginalia" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='marginalia': - c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "paragraph" in types_text: - if "type" in nn.attrib and nn.attrib['type']=='paragraph': - c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - + elif vv.tag!=link+'Point' and sumi>=1: break - if len(c_t_in_drop)>0: - co_text_drop.append(np.array(c_t_in_drop)) - if len(c_t_in_footnote_con)>0: - co_text_footnote_con.append(np.array(c_t_in_footnote_con)) - if len(c_t_in_footnote)>0: - co_text_footnote.append(np.array(c_t_in_footnote)) - if len(c_t_in_paragraph)>0: - co_text_paragraph.append(np.array(c_t_in_paragraph)) - if len(c_t_in_heading)>0: - co_text_heading.append(np.array(c_t_in_heading)) - - if len(c_t_in_header)>0: - co_text_header.append(np.array(c_t_in_header)) - if len(c_t_in_page_number)>0: - co_text_page_number.append(np.array(c_t_in_page_number)) - if len(c_t_in_catch)>0: - co_text_catch.append(np.array(c_t_in_catch)) - - if len(c_t_in_signature_mark)>0: - co_text_signature_mark.append(np.array(c_t_in_signature_mark)) - - if len(c_t_in_marginalia)>0: - co_text_marginalia.append(np.array(c_t_in_marginalia)) - - + for element_text in list(c_t_in.keys()): + if len(c_t_in[element_text])>0: + co_text[element_text].append(np.array(c_t_in[element_text])) + if 'graphicregions' in keys: if tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): #print('sth') for nn in root1.iter(tag): - c_t_in_stamp=[] - c_t_in_text_annotation=[] - c_t_in_decoration=[] - c_t_in_signature=[] + c_t_in_graphic = {"handwritten-annotation":[], "decoration":[], "stamp":[], "signature":[]} sumi=0 for vv in nn.iter(): # check the format of coords @@ -421,23 +319,22 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ coords=bool(vv.attrib) if coords: p_h=vv.attrib['points'].split(' ') - if "handwritten-annotation" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "decoration" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "stamp" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='stamp': - c_t_in_stamp.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - - if "signature" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='signature': - c_t_in_signature.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - + if "rest_as_decoration" in types_graphic: + types_graphic_without_decoration = [element for element in types_graphic if element!='rest_as_decoration' and element!='decoration'] + if len(types_graphic_without_decoration) == 0: + if "type" in nn.attrib: + c_t_in_graphic['decoration'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + elif len(types_graphic_without_decoration) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_graphic_without_decoration: + c_t_in_graphic[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in_graphic['decoration'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + else: + if "type" in nn.attrib: + c_t_in_graphic[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) break else: @@ -445,34 +342,33 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if vv.tag==link+'Point': - if "handwritten-annotation" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + if "rest_as_decoration" in types_graphic: + types_graphic_without_decoration = [element for element in types_graphic if element!='rest_as_decoration' and element!='decoration'] + if len(types_graphic_without_decoration) == 0: + if "type" in nn.attrib: + c_t_in_graphic['decoration'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + elif len(types_graphic_without_decoration) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_graphic_without_decoration: + c_t_in_graphic[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + else: + c_t_in_graphic['decoration'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + + else: + if "type" in nn.attrib: + c_t_in_graphic[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) sumi+=1 - if "decoration" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "stamp" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='stamp': - c_t_in_stamp.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 - - if "signature" in types_graphic: - if "type" in nn.attrib and nn.attrib['type']=='signature': - c_t_in_signature.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) - sumi+=1 + elif vv.tag!=link+'Point' and sumi>=1: + break + + for element_graphic in list(c_t_in_graphic.keys()): + if len(c_t_in_graphic[element_graphic])>0: + co_graphic[element_graphic].append(np.array(c_t_in_graphic[element_graphic])) - if len(c_t_in_text_annotation)>0: - co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) - if len(c_t_in_decoration)>0: - co_graphic_decoration.append(np.array(c_t_in_decoration)) - if len(c_t_in_stamp)>0: - co_graphic_stamp.append(np.array(c_t_in_stamp)) - if len(c_t_in_signature)>0: - co_graphic_signature.append(np.array(c_t_in_signature)) if 'imageregion' in keys: if tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): @@ -491,7 +387,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif vv.tag!=link+'Point' and sumi>=1: @@ -517,7 +413,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif vv.tag!=link+'Point' and sumi>=1: @@ -545,7 +441,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: @@ -571,7 +467,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: @@ -583,59 +479,63 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "paragraph" in elements_with_artificial_class: erosion_rate = 2 dilation_rate = 4 - co_text_paragraph, img_boundary = update_region_contours(co_text_paragraph, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text['paragraph'], img_boundary = update_region_contours(co_text['paragraph'], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "drop-capital" in elements_with_artificial_class: erosion_rate = 0 dilation_rate = 4 - co_text_drop, img_boundary = update_region_contours(co_text_drop, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["drop-capital"], img_boundary = update_region_contours(co_text["drop-capital"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "catch-word" in elements_with_artificial_class: erosion_rate = 0 dilation_rate = 4 - co_text_catch, img_boundary = update_region_contours(co_text_catch, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["catch-word"], img_boundary = update_region_contours(co_text["catch-word"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "page-number" in elements_with_artificial_class: erosion_rate = 0 dilation_rate = 4 - co_text_page_number, img_boundary = update_region_contours(co_text_page_number, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["page-number"], img_boundary = update_region_contours(co_text["page-number"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "header" in elements_with_artificial_class: erosion_rate = 1 dilation_rate = 4 - co_text_header, img_boundary = update_region_contours(co_text_header, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["header"], img_boundary = update_region_contours(co_text["header"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "heading" in elements_with_artificial_class: erosion_rate = 1 dilation_rate = 4 - co_text_heading, img_boundary = update_region_contours(co_text_heading, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["heading"], img_boundary = update_region_contours(co_text["heading"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "signature-mark" in elements_with_artificial_class: erosion_rate = 1 dilation_rate = 4 - co_text_signature_mark, img_boundary = update_region_contours(co_text_signature_mark, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["signature-mark"], img_boundary = update_region_contours(co_text["signature-mark"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "marginalia" in elements_with_artificial_class: erosion_rate = 2 dilation_rate = 4 - co_text_marginalia, img_boundary = update_region_contours(co_text_marginalia, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["marginalia"], img_boundary = update_region_contours(co_text["marginalia"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "footnote" in elements_with_artificial_class: erosion_rate = 2 dilation_rate = 4 - co_text_footnote, img_boundary = update_region_contours(co_text_footnote, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["footnote"], img_boundary = update_region_contours(co_text["footnote"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "footnote-continued" in elements_with_artificial_class: erosion_rate = 2 dilation_rate = 4 - co_text_footnote_con, img_boundary = update_region_contours(co_text_footnote_con, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + co_text["footnote-continued"], img_boundary = update_region_contours(co_text["footnote-continued"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) img = np.zeros( (y_len,x_len,3) ) if output_type == '3d': - if 'graphicregions' in keys: - if "handwritten-annotation" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=labels_rgb_color[ config_params['graphicregions']['handwritten-annotation']]) - if "signature" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=labels_rgb_color[ config_params['graphicregions']['signature']]) - if "decoration" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=labels_rgb_color[ config_params['graphicregions']['decoration']]) - if "stamp" in types_graphic: - img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=labels_rgb_color[ config_params['graphicregions']['stamp']]) + if 'rest_as_decoration' in types_graphic: + types_graphic[types_graphic=='rest_as_decoration'] = 'decoration' + for element_graphic in types_graphic: + if element_graphic == 'decoration': + color_label = labels_rgb_color[ config_params['graphicregions']['rest_as_decoration']] + else: + color_label = labels_rgb_color[ config_params['graphicregions'][element_graphic]] + img_poly=cv2.fillPoly(img, pts =co_graphic[element_graphic], color=color_label) + else: + for element_graphic in types_graphic: + color_label = labels_rgb_color[ config_params['graphicregions'][element_graphic]] + img_poly=cv2.fillPoly(img, pts =co_graphic[element_graphic], color=color_label) + if 'imageregion' in keys: img_poly=cv2.fillPoly(img, pts =co_img, color=labels_rgb_color[ config_params['imageregion']]) @@ -647,26 +547,19 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly=cv2.fillPoly(img, pts =co_noise, color=labels_rgb_color[ config_params['noiseregion']]) if 'textregions' in keys: - if "paragraph" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=labels_rgb_color[ config_params['textregions']['paragraph']]) - if "footnote" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=labels_rgb_color[ config_params['textregions']['footnote']]) - if "footnote-continued" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=labels_rgb_color[ config_params['textregions']['footnote-continued']]) - if "heading" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=labels_rgb_color[ config_params['textregions']['heading']]) - if "header" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_header, color=labels_rgb_color[ config_params['textregions']['header']]) - if "catch-word" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=labels_rgb_color[ config_params['textregions']['catch-word']]) - if "signature-mark" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=labels_rgb_color[ config_params['textregions']['signature-mark']]) - if "page-number" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=labels_rgb_color[ config_params['textregions']['page-number']]) - if "marginalia" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=labels_rgb_color[ config_params['textregions']['marginalia']]) - if "drop-capital" in types_text: - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=labels_rgb_color[ config_params['textregions']['drop-capital']]) + if 'rest_as_paragraph' in types_text: + types_text[types_text=='rest_as_paragraph'] = 'paragraph' + for element_text in types_text: + if element_text == 'paragraph': + color_label = labels_rgb_color[ config_params['textregions']['rest_as_paragraph']] + else: + color_label = labels_rgb_color[ config_params['textregions'][element_text]] + img_poly=cv2.fillPoly(img, pts =co_text[element_text], color=color_label) + else: + for element_text in types_text: + color_label = labels_rgb_color[ config_params['textregions'][element_text]] + img_poly=cv2.fillPoly(img, pts =co_text[element_text], color=color_label) + if "artificial_class_on_boundry" in keys: img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] @@ -678,18 +571,19 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ elif output_type == '2d': if 'graphicregions' in keys: - if "handwritten-annotation" in types_graphic: - color_label = config_params['graphicregions']['handwritten-annotation'] - img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(color_label,color_label,color_label)) - if "signature" in types_graphic: - color_label = config_params['graphicregions']['signature'] - img_poly=cv2.fillPoly(img, pts =co_graphic_signature, color=(color_label,color_label,color_label)) - if "decoration" in types_graphic: - color_label = config_params['graphicregions']['decoration'] - img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(color_label,color_label,color_label)) - if "stamp" in types_graphic: - color_label = config_params['graphicregions']['stamp'] - img_poly=cv2.fillPoly(img, pts =co_graphic_stamp, color=(color_label,color_label,color_label)) + if 'rest_as_decoration' in types_graphic: + types_graphic[types_graphic=='rest_as_decoration'] = 'decoration' + for element_graphic in types_graphic: + if element_graphic == 'decoration': + color_label = config_params['graphicregions']['rest_as_decoration'] + else: + color_label = config_params['graphicregions'][element_graphic] + img_poly=cv2.fillPoly(img, pts =co_graphic[element_graphic], color=color_label) + else: + for element_graphic in types_graphic: + color_label = config_params['graphicregions'][element_graphic] + img_poly=cv2.fillPoly(img, pts =co_graphic[element_graphic], color=color_label) + if 'imageregion' in keys: color_label = config_params['imageregion'] @@ -705,36 +599,18 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly=cv2.fillPoly(img, pts =co_noise, color=(color_label,color_label,color_label)) if 'textregions' in keys: - if "paragraph" in types_text: - color_label = config_params['textregions']['paragraph'] - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(color_label,color_label,color_label)) - if "footnote" in types_text: - color_label = config_params['textregions']['footnote'] - img_poly=cv2.fillPoly(img, pts =co_text_footnote, color=(color_label,color_label,color_label)) - if "footnote-continued" in types_text: - color_label = config_params['textregions']['footnote-continued'] - img_poly=cv2.fillPoly(img, pts =co_text_footnote_con, color=(color_label,color_label,color_label)) - if "heading" in types_text: - color_label = config_params['textregions']['heading'] - img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(color_label,color_label,color_label)) - if "header" in types_text: - color_label = config_params['textregions']['header'] - img_poly=cv2.fillPoly(img, pts =co_text_header, color=(color_label,color_label,color_label)) - if "catch-word" in types_text: - color_label = config_params['textregions']['catch-word'] - img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(color_label,color_label,color_label)) - if "signature-mark" in types_text: - color_label = config_params['textregions']['signature-mark'] - img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(color_label,color_label,color_label)) - if "page-number" in types_text: - color_label = config_params['textregions']['page-number'] - img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(color_label,color_label,color_label)) - if "marginalia" in types_text: - color_label = config_params['textregions']['marginalia'] - img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(color_label,color_label,color_label)) - if "drop-capital" in types_text: - color_label = config_params['textregions']['drop-capital'] - img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(color_label,color_label,color_label)) + if 'rest_as_paragraph' in types_text: + types_text[types_text=='rest_as_paragraph'] = 'paragraph' + for element_text in types_text: + if element_text == 'paragraph': + color_label = config_params['textregions']['rest_as_paragraph'] + else: + color_label = config_params['textregions'][element_text] + img_poly=cv2.fillPoly(img, pts =co_text[element_text], color=color_label) + else: + for element_text in types_text: + color_label = config_params['textregions'][element_text] + img_poly=cv2.fillPoly(img, pts =co_text[element_text], color=color_label) if "artificial_class_on_boundry" in keys: img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label @@ -947,51 +823,51 @@ def read_xml(xml_file): if "type" in nn.attrib and nn.attrib['type']=='drop-capital': #if nn.attrib['type']=='paragraph': - c_t_in_drop.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_drop.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='heading': id_heading.append(nn.attrib['id']) - c_t_in_heading.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_heading.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': - c_t_in_signature_mark.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_signature_mark.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) #print(c_t_in_paragraph) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='header': id_header.append(nn.attrib['id']) - c_t_in_header.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_header.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_catch.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='page-number': - c_t_in_page_number.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_page_number.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) #print(c_t_in_paragraph) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='marginalia': id_marginalia.append(nn.attrib['id']) - c_t_in_marginalia.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_marginalia.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) #print(c_t_in_paragraph) sumi+=1 else: id_paragraph.append(nn.attrib['id']) - c_t_in_paragraph.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_paragraph.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) #print(c_t_in_paragraph) sumi+=1 - #c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + #c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: @@ -1057,16 +933,16 @@ def read_xml(xml_file): if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': #if nn.attrib['type']=='paragraph': - c_t_in_text_annotation.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_text_annotation.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in_decoration.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) #print(c_t_in_paragraph) sumi+=1 else: - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 if len(c_t_in_text_annotation)>0: @@ -1096,7 +972,7 @@ def read_xml(xml_file): if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: @@ -1123,7 +999,7 @@ def read_xml(xml_file): if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: @@ -1150,7 +1026,7 @@ def read_xml(xml_file): if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: @@ -1176,7 +1052,7 @@ def read_xml(xml_file): if vv.tag==link+'Point': - c_t_in.append([ int(np.float(vv.attrib['x'])) , int(np.float(vv.attrib['y'])) ]) + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: From 467bbb2884e1b900e819370b1e88853c24d60e90 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 28 May 2024 10:01:17 +0200 Subject: [PATCH 057/374] pass degrading scales for image enhancement as a json file --- train/generate_gt_for_training.py | 16 ++++++++++------ train/scales_enhancement.json | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 train/scales_enhancement.json diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index e296029..2a2a776 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -64,13 +64,17 @@ def pagexml2label(dir_xml,dir_out,type_output,config): help="directory where original images will be written as labels.", type=click.Path(exists=True, file_okay=False), ) -def image_enhancement(dir_imgs, dir_out_images, dir_out_labels): - #dir_imgs = './training_data_sample_enhancement/images' - #dir_out_images = './training_data_sample_enhancement/images_gt' - #dir_out_labels = './training_data_sample_enhancement/labels_gt' - +@click.option( + "--scales", + "-scs", + help="json dictionary where the scales are written.", + type=click.Path(exists=True, dir_okay=False), +) +def image_enhancement(dir_imgs, dir_out_images, dir_out_labels, scales): ls_imgs = os.listdir(dir_imgs) - ls_scales = [ 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] + with open(scales) as f: + scale_dict = json.load(f) + ls_scales = scale_dict['scales'] for img in tqdm(ls_imgs): img_name = img.split('.')[0] diff --git a/train/scales_enhancement.json b/train/scales_enhancement.json new file mode 100644 index 0000000..58034f0 --- /dev/null +++ b/train/scales_enhancement.json @@ -0,0 +1,3 @@ +{ + "scales" : [ 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] +} From cc7577d2c121ca14180bbc732355e35d7be80af8 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 28 May 2024 10:14:16 +0200 Subject: [PATCH 058/374] min area size of text region passes as an argument for machine based reading order --- train/generate_gt_for_training.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 2a2a776..cf2b2a6 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -116,22 +116,28 @@ def image_enhancement(dir_imgs, dir_out_images, dir_out_labels, scales): @click.option( "--input_height", "-ih", - help="input_height", + help="input height", ) @click.option( "--input_width", "-iw", - help="input_width", + help="input width", +) +@click.option( + "--min_area_size", + "-min", + help="min area size of regions considered for reading order training.", ) -def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width): +def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width, min_area_size): xml_files_ind = os.listdir(dir_xml) input_height = int(input_height) input_width = int(input_width) + min_area = float(min_area_size) indexer_start= 0#55166 max_area = 1 - min_area = 0.0001 + #min_area = 0.0001 for ind_xml in tqdm(xml_files_ind): indexer = 0 From 4fb45a671114c8d44b100dd799e097a3b669c27a Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 28 May 2024 16:48:51 +0200 Subject: [PATCH 059/374] inference for reading order --- train/gt_gen_utils.py | 134 +++++++++-------------------- train/inference.py | 196 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 227 insertions(+), 103 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 9dc8377..0286ac7 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -38,11 +38,8 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m polygon = geometry.Polygon([point[0] for point in c]) # area = cv2.contourArea(c) area = polygon.area - ##print(np.prod(thresh.shape[:2])) # Check that polygon has area greater than minimal area - # print(hierarchy[0][jv][3],hierarchy ) if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : - # print(c[0][0][1]) found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.int32)) jv += 1 return found_polygons_early @@ -52,15 +49,12 @@ def filter_contours_area_of_image(image, contours, order_index, max_area, min_ar order_index_filtered = list() #jv = 0 for jv, c in enumerate(contours): - #print(len(c[0])) c = c[0] if len(c) < 3: # A polygon cannot have less than 3 points continue c_e = [point for point in c] - #print(c_e) polygon = geometry.Polygon(c_e) area = polygon.area - #print(area,'area') if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.uint)) order_index_filtered.append(order_index[jv]) @@ -88,12 +82,8 @@ def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len): co_text_eroded = [] for con in co_text: - #try: img_boundary_in = np.zeros( (y_len,x_len) ) img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) - #print('bidiahhhhaaa') - - #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica if erosion_rate > 0: @@ -626,8 +616,6 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ def find_new_features_of_contours(contours_main): - - #print(contours_main[0][0][:, 0]) areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] @@ -658,8 +646,6 @@ def find_new_features_of_contours(contours_main): y_min_main = np.array([np.min(contours_main[j][:, 1]) for j in range(len(contours_main))]) y_max_main = np.array([np.max(contours_main[j][:, 1]) for j in range(len(contours_main))]) - # dis_x=np.abs(x_max_main-x_min_main) - return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin def read_xml(xml_file): file_name = Path(xml_file).stem @@ -675,13 +661,11 @@ def read_xml(xml_file): y_len=int(jj.attrib['imageHeight']) x_len=int(jj.attrib['imageWidth']) - for jj in root1.iter(link+'RegionRefIndexed'): index_tot_regions.append(jj.attrib['index']) tot_region_ref.append(jj.attrib['regionRef']) region_tags=np.unique([x for x in alltags if x.endswith('Region')]) - #print(region_tags) co_text_paragraph=[] co_text_drop=[] co_text_heading=[] @@ -698,7 +682,6 @@ def read_xml(xml_file): co_graphic_decoration=[] co_noise=[] - co_text_paragraph_text=[] co_text_drop_text=[] co_text_heading_text=[] @@ -715,7 +698,6 @@ def read_xml(xml_file): co_graphic_decoration_text=[] co_noise_text=[] - id_paragraph = [] id_header = [] id_heading = [] @@ -726,14 +708,8 @@ def read_xml(xml_file): for nn in root1.iter(tag): for child2 in nn: tag2 = child2.tag - #print(child2.tag) if tag2.endswith('}TextEquiv') or tag2.endswith('}TextEquiv'): - #children2 = childtext.getchildren() - #rank = child2.find('Unicode').text for childtext2 in child2: - #rank = childtext2.find('Unicode').text - #if childtext2.tag.endswith('}PlainText') or childtext2.tag.endswith('}PlainText'): - #print(childtext2.text) if childtext2.tag.endswith('}Unicode') or childtext2.tag.endswith('}Unicode'): if "type" in nn.attrib and nn.attrib['type']=='drop-capital': co_text_drop_text.append(childtext2.text) @@ -743,10 +719,10 @@ def read_xml(xml_file): co_text_signature_mark_text.append(childtext2.text) elif "type" in nn.attrib and nn.attrib['type']=='header': co_text_header_text.append(childtext2.text) - elif "type" in nn.attrib and nn.attrib['type']=='catch-word': - co_text_catch_text.append(childtext2.text) - elif "type" in nn.attrib and nn.attrib['type']=='page-number': - co_text_page_number_text.append(childtext2.text) + ###elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + ###co_text_catch_text.append(childtext2.text) + ###elif "type" in nn.attrib and nn.attrib['type']=='page-number': + ###co_text_page_number_text.append(childtext2.text) elif "type" in nn.attrib and nn.attrib['type']=='marginalia': co_text_marginalia_text.append(childtext2.text) else: @@ -774,7 +750,6 @@ def read_xml(xml_file): if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - #if nn.attrib['type']=='paragraph': c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) @@ -792,27 +767,22 @@ def read_xml(xml_file): c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - elif "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + ###elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + ###c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - elif "type" in nn.attrib and nn.attrib['type']=='page-number': + ###elif "type" in nn.attrib and nn.attrib['type']=='page-number': - c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) + ###c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) elif "type" in nn.attrib and nn.attrib['type']=='marginalia': id_marginalia.append(nn.attrib['id']) c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) else: - #print(nn.attrib['id']) - id_paragraph.append(nn.attrib['id']) c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) break else: @@ -821,7 +791,6 @@ def read_xml(xml_file): if vv.tag==link+'Point': if "type" in nn.attrib and nn.attrib['type']=='drop-capital': - #if nn.attrib['type']=='paragraph': c_t_in_drop.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 @@ -835,7 +804,6 @@ def read_xml(xml_file): elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': c_t_in_signature_mark.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='header': id_header.append(nn.attrib['id']) @@ -843,33 +811,26 @@ def read_xml(xml_file): sumi+=1 - elif "type" in nn.attrib and nn.attrib['type']=='catch-word': - c_t_in_catch.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - sumi+=1 + ###elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + ###c_t_in_catch.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + ###sumi+=1 + ###elif "type" in nn.attrib and nn.attrib['type']=='page-number': - elif "type" in nn.attrib and nn.attrib['type']=='page-number': - - c_t_in_page_number.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) - sumi+=1 + ###c_t_in_page_number.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + ###sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='marginalia': id_marginalia.append(nn.attrib['id']) c_t_in_marginalia.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) sumi+=1 else: id_paragraph.append(nn.attrib['id']) c_t_in_paragraph.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) sumi+=1 - #c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - - #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: break @@ -895,7 +856,6 @@ def read_xml(xml_file): elif tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): - #print('sth') for nn in root1.iter(tag): c_t_in=[] c_t_in_text_annotation=[] @@ -907,40 +867,31 @@ def read_xml(xml_file): coords=bool(vv.attrib) if coords: p_h=vv.attrib['points'].split(' ') - #c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - #if nn.attrib['type']=='paragraph': - c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - + elif "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - #print(c_t_in_paragraph) + else: c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) - break else: pass if vv.tag==link+'Point': - if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': - #if nn.attrib['type']=='paragraph': - c_t_in_text_annotation.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='decoration': - c_t_in_decoration.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) - #print(c_t_in_paragraph) sumi+=1 + else: c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 @@ -955,7 +906,6 @@ def read_xml(xml_file): elif tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): - #print('sth') for nn in root1.iter(tag): c_t_in=[] sumi=0 @@ -974,7 +924,6 @@ def read_xml(xml_file): if vv.tag==link+'Point': c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 - #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: break co_img.append(np.array(c_t_in)) @@ -982,7 +931,6 @@ def read_xml(xml_file): elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): - #print('sth') for nn in root1.iter(tag): c_t_in=[] sumi=0 @@ -1001,7 +949,6 @@ def read_xml(xml_file): if vv.tag==link+'Point': c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 - #print(vv.tag,'in') elif vv.tag!=link+'Point' and sumi>=1: break co_sep.append(np.array(c_t_in)) @@ -1009,7 +956,6 @@ def read_xml(xml_file): elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): - #print('sth') for nn in root1.iter(tag): c_t_in=[] sumi=0 @@ -1028,14 +974,13 @@ def read_xml(xml_file): if vv.tag==link+'Point': c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 - #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: break co_table.append(np.array(c_t_in)) co_table_text.append(' ') elif tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): - #print('sth') for nn in root1.iter(tag): c_t_in=[] sumi=0 @@ -1054,40 +999,22 @@ def read_xml(xml_file): if vv.tag==link+'Point': c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 - #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: break co_noise.append(np.array(c_t_in)) co_noise_text.append(' ') - img = np.zeros( (y_len,x_len,3) ) - img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(1,1,1)) img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(2,2,2)) img_poly=cv2.fillPoly(img, pts =co_text_header, color=(2,2,2)) - #img_poly=cv2.fillPoly(img, pts =co_text_catch, color=(125,255,125)) - #img_poly=cv2.fillPoly(img, pts =co_text_signature_mark, color=(125,125,0)) - #img_poly=cv2.fillPoly(img, pts =co_graphic_decoration, color=(1,125,255)) - #img_poly=cv2.fillPoly(img, pts =co_text_page_number, color=(1,125,0)) img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(3,3,3)) - #img_poly=cv2.fillPoly(img, pts =co_text_drop, color=(1,125,255)) - - #img_poly=cv2.fillPoly(img, pts =co_graphic_text_annotation, color=(125,0,125)) img_poly=cv2.fillPoly(img, pts =co_img, color=(4,4,4)) img_poly=cv2.fillPoly(img, pts =co_sep, color=(5,5,5)) - #img_poly=cv2.fillPoly(img, pts =co_table, color=(1,255,255)) - #img_poly=cv2.fillPoly(img, pts =co_graphic, color=(255,125,125)) - #img_poly=cv2.fillPoly(img, pts =co_noise, color=(255,0,255)) - #print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.jpg') - ###try: - ####print('yazdimmm',self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.jpg') - ###cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('-')[1].split('.')[0]+'.jpg',img_poly ) - ###except: - ###cv2.imwrite(self.output_dir+'/'+self.gt_list[index].split('.')[0]+'.jpg',img_poly ) - return file_name, id_paragraph, id_header,co_text_paragraph, co_text_header,\ + return tree1, root1, file_name, id_paragraph, id_header,co_text_paragraph, co_text_header,\ tot_region_ref,x_len, y_len,index_tot_regions, img_poly @@ -1113,3 +1040,24 @@ def make_image_from_bb(width_l, height_l, bb_all): for i in range(bb_all.shape[0]): img_remade[bb_all[i,1]:bb_all[i,1]+bb_all[i,3],bb_all[i,0]:bb_all[i,0]+bb_all[i,2] ] = 1 return img_remade + +def update_list_and_return_first_with_length_bigger_than_one(index_element_to_be_updated, innner_index_pr_pos, pr_list, pos_list,list_inp): + list_inp.pop(index_element_to_be_updated) + if len(pr_list)>0: + list_inp.insert(index_element_to_be_updated, pr_list) + else: + index_element_to_be_updated = index_element_to_be_updated -1 + + list_inp.insert(index_element_to_be_updated+1, [innner_index_pr_pos]) + if len(pos_list)>0: + list_inp.insert(index_element_to_be_updated+2, pos_list) + + len_all_elements = [len(i) for i in list_inp] + list_len_bigger_1 = np.where(np.array(len_all_elements)>1) + list_len_bigger_1 = list_len_bigger_1[0] + + if len(list_len_bigger_1)>0: + early_list_bigger_than_one = list_len_bigger_1[0] + else: + early_list_bigger_than_one = -20 + return list_inp, early_list_bigger_than_one diff --git a/train/inference.py b/train/inference.py index 94e318d..73b4ed8 100644 --- a/train/inference.py +++ b/train/inference.py @@ -11,13 +11,11 @@ from tensorflow.keras import layers import tensorflow.keras.losses from tensorflow.keras.layers import * from models import * +from gt_gen_utils import * import click import json from tensorflow.python.keras import backend as tensorflow_backend - - - - +import xml.etree.ElementTree as ET with warnings.catch_warnings(): @@ -29,7 +27,7 @@ Tool to load model and predict for given image. """ class sbb_predict: - def __init__(self,image, model, task, config_params_model, patches, save, ground_truth): + def __init__(self,image, model, task, config_params_model, patches, save, ground_truth, xml_file): self.image=image self.patches=patches self.save=save @@ -37,6 +35,7 @@ class sbb_predict: self.ground_truth=ground_truth self.task=task self.config_params_model=config_params_model + self.xml_file = xml_file def resize_image(self,img_in,input_height,input_width): return cv2.resize( img_in, ( input_width,input_height) ,interpolation=cv2.INTER_NEAREST) @@ -166,7 +165,7 @@ class sbb_predict: ##if self.weights_dir!=None: ##self.model.load_weights(self.weights_dir) - if self.task != 'classification': + if (self.task != 'classification' and self.task != 'reading_order'): self.img_height=self.model.layers[len(self.model.layers)-1].output_shape[1] self.img_width=self.model.layers[len(self.model.layers)-1].output_shape[2] self.n_classes=self.model.layers[len(self.model.layers)-1].output_shape[3] @@ -233,6 +232,178 @@ class sbb_predict: index_class = np.argmax(label_p_pred[0]) print("Predicted Class: {}".format(classes_names[str(int(index_class))])) + elif self.task == 'reading_order': + img_height = self.config_params_model['input_height'] + img_width = self.config_params_model['input_width'] + + tree_xml, root_xml, file_name, id_paragraph, id_header, co_text_paragraph, co_text_header, tot_region_ref, x_len, y_len, index_tot_regions, img_poly = read_xml(self.xml_file) + _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours(co_text_header) + + img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') + + for j in range(len(cy_main)): + img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12,int(x_min_main[j]):int(x_max_main[j]) ] = 1 + + co_text_all = co_text_paragraph + co_text_header + id_all_text = id_paragraph + id_header + + ##texts_corr_order_index = [index_tot_regions[tot_region_ref.index(i)] for i in id_all_text ] + ##texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] + texts_corr_order_index_int = list(np.array(range(len(co_text_all)))) + + min_area = 0 + max_area = 1 + + co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area) + + labels_con = np.zeros((y_len,x_len,len(co_text_all)),dtype='uint8') + for i in range(len(co_text_all)): + img_label = np.zeros((y_len,x_len,3),dtype='uint8') + img_label=cv2.fillPoly(img_label, pts =[co_text_all[i]], color=(1,1,1)) + labels_con[:,:,i] = img_label[:,:,0] + + img3= np.copy(img_poly) + labels_con = resize_image(labels_con, img_height, img_width) + + img_header_and_sep = resize_image(img_header_and_sep, img_height, img_width) + + img3= resize_image (img3, img_height, img_width) + img3 = img3.astype(np.uint16) + + inference_bs = 1#4 + + input_1= np.zeros( (inference_bs, img_height, img_width,3)) + + + starting_list_of_regions = [] + starting_list_of_regions.append( list(range(labels_con.shape[2])) ) + + index_update = 0 + index_selected = starting_list_of_regions[0] + + scalibility_num = 0 + while index_update>=0: + ij_list = starting_list_of_regions[index_update] + i = ij_list[0] + ij_list.pop(0) + + + pr_list = [] + post_list = [] + + batch_counter = 0 + tot_counter = 1 + + tot_iteration = len(ij_list) + full_bs_ite= tot_iteration//inference_bs + last_bs = tot_iteration % inference_bs + + jbatch_indexer =[] + for j in ij_list: + img1= np.repeat(labels_con[:,:,i][:, :, np.newaxis], 3, axis=2) + img2 = np.repeat(labels_con[:,:,j][:, :, np.newaxis], 3, axis=2) + + + img2[:,:,0][img3[:,:,0]==5] = 2 + img2[:,:,0][img_header_and_sep[:,:]==1] = 3 + + + + img1[:,:,0][img3[:,:,0]==5] = 2 + img1[:,:,0][img_header_and_sep[:,:]==1] = 3 + + #input_1= np.zeros( (height1, width1,3)) + + + jbatch_indexer.append(j) + + input_1[batch_counter,:,:,0] = img1[:,:,0]/3. + input_1[batch_counter,:,:,2] = img2[:,:,0]/3. + input_1[batch_counter,:,:,1] = img3[:,:,0]/5. + #input_1[batch_counter,:,:,:]= np.zeros( (batch_counter, height1, width1,3)) + batch_counter = batch_counter+1 + + #input_1[:,:,0] = img1[:,:,0]/3. + #input_1[:,:,2] = img2[:,:,0]/3. + #input_1[:,:,1] = img3[:,:,0]/5. + + if batch_counter==inference_bs or ( (tot_counter//inference_bs)==full_bs_ite and tot_counter%inference_bs==last_bs): + y_pr = self.model.predict(input_1 , verbose=0) + scalibility_num = scalibility_num+1 + + if batch_counter==inference_bs: + iteration_batches = inference_bs + else: + iteration_batches = last_bs + for jb in range(iteration_batches): + if y_pr[jb][0]>=0.5: + post_list.append(jbatch_indexer[jb]) + else: + pr_list.append(jbatch_indexer[jb]) + + batch_counter = 0 + jbatch_indexer = [] + + tot_counter = tot_counter+1 + + starting_list_of_regions, index_update = update_list_and_return_first_with_length_bigger_than_one(index_update, i, pr_list, post_list,starting_list_of_regions) + + index_sort = [i[0] for i in starting_list_of_regions ] + + + alltags=[elem.tag for elem in root_xml.iter()] + + + + link=alltags[0].split('}')[0]+'}' + name_space = alltags[0].split('}')[0] + name_space = name_space.split('{')[1] + + page_element = root_xml.find(link+'Page') + + """ + ro_subelement = ET.SubElement(page_element, 'ReadingOrder') + #print(page_element, 'page_element') + + #new_element = ET.Element('ReadingOrder') + + new_element_element = ET.Element('OrderedGroup') + new_element_element.set('id', "ro357564684568544579089") + + for index, id_text in enumerate(id_all_text): + new_element_2 = ET.Element('RegionRefIndexed') + new_element_2.set('regionRef', id_all_text[index]) + new_element_2.set('index', str(index_sort[index])) + + new_element_element.append(new_element_2) + + ro_subelement.append(new_element_element) + """ + ##ro_subelement = ET.SubElement(page_element, 'ReadingOrder') + + ro_subelement = ET.Element('ReadingOrder') + + ro_subelement2 = ET.SubElement(ro_subelement, 'OrderedGroup') + ro_subelement2.set('id', "ro357564684568544579089") + + for index, id_text in enumerate(id_all_text): + new_element_2 = ET.SubElement(ro_subelement2, 'RegionRefIndexed') + new_element_2.set('regionRef', id_all_text[index]) + new_element_2.set('index', str(index_sort[index])) + + if link+'PrintSpace' in alltags: + page_element.insert(1, ro_subelement) + else: + page_element.insert(0, ro_subelement) + + #page_element[0].append(new_element) + #root_xml.append(new_element) + alltags=[elem.tag for elem in root_xml.iter()] + + ET.register_namespace("",name_space) + tree_xml.write('library2.xml',xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) + #tree_xml.write('library2.xml') + else: if self.patches: #def textline_contours(img,input_width,input_height,n_classes,model): @@ -356,7 +527,7 @@ class sbb_predict: def run(self): res=self.predict() - if self.task == 'classification': + if (self.task == 'classification' or self.task == 'reading_order'): pass else: img_seg_overlayed = self.visualize_model_output(res, self.img_org, self.task) @@ -397,15 +568,20 @@ class sbb_predict: "-gt", help="ground truth directory if you want to see the iou of prediction.", ) -def main(image, model, patches, save, ground_truth): +@click.option( + "--xml_file", + "-xml", + help="xml file with layout coordinates that reading order detection will be implemented on. The result will be written in the same xml file.", +) +def main(image, model, patches, save, ground_truth, xml_file): with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) task = config_params_model['task'] - if task != 'classification': + if (task != 'classification' and task != 'reading_order'): if not save: print("Error: You used one of segmentation or binarization task but not set -s, you need a filename to save visualized output with -s") sys.exit(1) - x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth) + x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, xml_file) x.run() if __name__=="__main__": From 06ed00619399fb93d48bd803f4bd66ba942d4d84 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 29 May 2024 11:18:35 +0200 Subject: [PATCH 060/374] reading order detection on xml with layout + result will be written in an output directory with the same file name --- train/gt_gen_utils.py | 74 +++++++++++++++++++++++++++++++++++++------ train/inference.py | 45 +++++++++++++++++++------- 2 files changed, 99 insertions(+), 20 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 0286ac7..8f72fb8 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -664,6 +664,58 @@ def read_xml(xml_file): for jj in root1.iter(link+'RegionRefIndexed'): index_tot_regions.append(jj.attrib['index']) tot_region_ref.append(jj.attrib['regionRef']) + + if (link+'PrintSpace' in alltags) or (link+'Border' in alltags): + co_printspace = [] + if link+'PrintSpace' in alltags: + region_tags_printspace = np.unique([x for x in alltags if x.endswith('PrintSpace')]) + elif link+'Border' in alltags: + region_tags_printspace = np.unique([x for x in alltags if x.endswith('Border')]) + + for tag in region_tags_printspace: + if link+'PrintSpace' in alltags: + tag_endings_printspace = ['}PrintSpace','}printspace'] + elif link+'Border' in alltags: + tag_endings_printspace = ['}Border','}border'] + + if tag.endswith(tag_endings_printspace[0]) or tag.endswith(tag_endings_printspace[1]): + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + # check the format of coords + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_printspace.append(np.array(c_t_in)) + img_printspace = np.zeros( (y_len,x_len,3) ) + img_printspace=cv2.fillPoly(img_printspace, pts =co_printspace, color=(1,1,1)) + img_printspace = img_printspace.astype(np.uint8) + + imgray = cv2.cvtColor(img_printspace, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) + cnt = contours[np.argmax(cnt_size)] + x, y, w, h = cv2.boundingRect(cnt) + + bb_coord_printspace = [x, y, w, h] + + else: + bb_coord_printspace = None + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) co_text_paragraph=[] @@ -754,7 +806,7 @@ def read_xml(xml_file): c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) elif "type" in nn.attrib and nn.attrib['type']=='heading': - id_heading.append(nn.attrib['id']) + ##id_heading.append(nn.attrib['id']) c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) @@ -763,7 +815,7 @@ def read_xml(xml_file): c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) #print(c_t_in_paragraph) elif "type" in nn.attrib and nn.attrib['type']=='header': - id_header.append(nn.attrib['id']) + #id_header.append(nn.attrib['id']) c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) @@ -776,11 +828,11 @@ def read_xml(xml_file): ###c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) elif "type" in nn.attrib and nn.attrib['type']=='marginalia': - id_marginalia.append(nn.attrib['id']) + #id_marginalia.append(nn.attrib['id']) c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) else: - id_paragraph.append(nn.attrib['id']) + #id_paragraph.append(nn.attrib['id']) c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) @@ -796,7 +848,7 @@ def read_xml(xml_file): sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='heading': - id_heading.append(nn.attrib['id']) + #id_heading.append(nn.attrib['id']) c_t_in_heading.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 @@ -806,7 +858,7 @@ def read_xml(xml_file): c_t_in_signature_mark.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='header': - id_header.append(nn.attrib['id']) + #id_header.append(nn.attrib['id']) c_t_in_header.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 @@ -821,13 +873,13 @@ def read_xml(xml_file): ###sumi+=1 elif "type" in nn.attrib and nn.attrib['type']=='marginalia': - id_marginalia.append(nn.attrib['id']) + #id_marginalia.append(nn.attrib['id']) c_t_in_marginalia.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 else: - id_paragraph.append(nn.attrib['id']) + #id_paragraph.append(nn.attrib['id']) c_t_in_paragraph.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) sumi+=1 @@ -838,11 +890,14 @@ def read_xml(xml_file): co_text_drop.append(np.array(c_t_in_drop)) if len(c_t_in_paragraph)>0: co_text_paragraph.append(np.array(c_t_in_paragraph)) + id_paragraph.append(nn.attrib['id']) if len(c_t_in_heading)>0: co_text_heading.append(np.array(c_t_in_heading)) + id_heading.append(nn.attrib['id']) if len(c_t_in_header)>0: co_text_header.append(np.array(c_t_in_header)) + id_header.append(nn.attrib['id']) if len(c_t_in_page_number)>0: co_text_page_number.append(np.array(c_t_in_page_number)) if len(c_t_in_catch)>0: @@ -853,6 +908,7 @@ def read_xml(xml_file): if len(c_t_in_marginalia)>0: co_text_marginalia.append(np.array(c_t_in_marginalia)) + id_marginalia.append(nn.attrib['id']) elif tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): @@ -1014,7 +1070,7 @@ def read_xml(xml_file): img_poly=cv2.fillPoly(img, pts =co_img, color=(4,4,4)) img_poly=cv2.fillPoly(img, pts =co_sep, color=(5,5,5)) - return tree1, root1, file_name, id_paragraph, id_header,co_text_paragraph, co_text_header,\ + return tree1, root1, bb_coord_printspace, file_name, id_paragraph, id_header+id_heading, co_text_paragraph, co_text_header+co_text_heading,\ tot_region_ref,x_len, y_len,index_tot_regions, img_poly diff --git a/train/inference.py b/train/inference.py index 73b4ed8..28445e8 100644 --- a/train/inference.py +++ b/train/inference.py @@ -16,6 +16,7 @@ import click import json from tensorflow.python.keras import backend as tensorflow_backend import xml.etree.ElementTree as ET +import matplotlib.pyplot as plt with warnings.catch_warnings(): @@ -27,7 +28,7 @@ Tool to load model and predict for given image. """ class sbb_predict: - def __init__(self,image, model, task, config_params_model, patches, save, ground_truth, xml_file): + def __init__(self,image, model, task, config_params_model, patches, save, ground_truth, xml_file, out): self.image=image self.patches=patches self.save=save @@ -36,6 +37,7 @@ class sbb_predict: self.task=task self.config_params_model=config_params_model self.xml_file = xml_file + self.out = out def resize_image(self,img_in,input_height,input_width): return cv2.resize( img_in, ( input_width,input_height) ,interpolation=cv2.INTER_NEAREST) @@ -236,16 +238,18 @@ class sbb_predict: img_height = self.config_params_model['input_height'] img_width = self.config_params_model['input_width'] - tree_xml, root_xml, file_name, id_paragraph, id_header, co_text_paragraph, co_text_header, tot_region_ref, x_len, y_len, index_tot_regions, img_poly = read_xml(self.xml_file) + tree_xml, root_xml, bb_coord_printspace, file_name, id_paragraph, id_header, co_text_paragraph, co_text_header, tot_region_ref, x_len, y_len, index_tot_regions, img_poly = read_xml(self.xml_file) _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours(co_text_header) img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') + for j in range(len(cy_main)): img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12,int(x_min_main[j]):int(x_max_main[j]) ] = 1 co_text_all = co_text_paragraph + co_text_header id_all_text = id_paragraph + id_header + ##texts_corr_order_index = [index_tot_regions[tot_region_ref.index(i)] for i in id_all_text ] ##texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] @@ -253,8 +257,9 @@ class sbb_predict: min_area = 0 max_area = 1 + - co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area) + ##co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area) labels_con = np.zeros((y_len,x_len,len(co_text_all)),dtype='uint8') for i in range(len(co_text_all)): @@ -262,6 +267,18 @@ class sbb_predict: img_label=cv2.fillPoly(img_label, pts =[co_text_all[i]], color=(1,1,1)) labels_con[:,:,i] = img_label[:,:,0] + if bb_coord_printspace: + #bb_coord_printspace[x,y,w,h,_,_] + x = bb_coord_printspace[0] + y = bb_coord_printspace[1] + w = bb_coord_printspace[2] + h = bb_coord_printspace[3] + labels_con = labels_con[y:y+h, x:x+w, :] + img_poly = img_poly[y:y+h, x:x+w, :] + img_header_and_sep = img_header_and_sep[y:y+h, x:x+w] + + + img3= np.copy(img_poly) labels_con = resize_image(labels_con, img_height, img_width) @@ -347,9 +364,11 @@ class sbb_predict: tot_counter = tot_counter+1 starting_list_of_regions, index_update = update_list_and_return_first_with_length_bigger_than_one(index_update, i, pr_list, post_list,starting_list_of_regions) - + + index_sort = [i[0] for i in starting_list_of_regions ] + id_all_text = np.array(id_all_text)[index_sort] alltags=[elem.tag for elem in root_xml.iter()] @@ -389,19 +408,17 @@ class sbb_predict: for index, id_text in enumerate(id_all_text): new_element_2 = ET.SubElement(ro_subelement2, 'RegionRefIndexed') new_element_2.set('regionRef', id_all_text[index]) - new_element_2.set('index', str(index_sort[index])) + new_element_2.set('index', str(index)) - if link+'PrintSpace' in alltags: + if (link+'PrintSpace' in alltags) or (link+'Border' in alltags): page_element.insert(1, ro_subelement) else: page_element.insert(0, ro_subelement) - #page_element[0].append(new_element) - #root_xml.append(new_element) alltags=[elem.tag for elem in root_xml.iter()] ET.register_namespace("",name_space) - tree_xml.write('library2.xml',xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) + tree_xml.write(os.path.join(self.out, file_name+'.xml'),xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) #tree_xml.write('library2.xml') else: @@ -545,6 +562,12 @@ class sbb_predict: help="image filename", type=click.Path(exists=True, dir_okay=False), ) +@click.option( + "--out", + "-o", + help="output directory where xml with detected reading order will be written.", + type=click.Path(exists=True, file_okay=False), +) @click.option( "--patches/--no-patches", "-p/-nop", @@ -573,7 +596,7 @@ class sbb_predict: "-xml", help="xml file with layout coordinates that reading order detection will be implemented on. The result will be written in the same xml file.", ) -def main(image, model, patches, save, ground_truth, xml_file): +def main(image, model, patches, save, ground_truth, xml_file, out): with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) task = config_params_model['task'] @@ -581,7 +604,7 @@ def main(image, model, patches, save, ground_truth, xml_file): if not save: print("Error: You used one of segmentation or binarization task but not set -s, you need a filename to save visualized output with -s") sys.exit(1) - x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, xml_file) + x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, xml_file, out) x.run() if __name__=="__main__": From 09789619a8fe9589352f7bde6c0e7cb41a9ea087 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 29 May 2024 13:07:06 +0200 Subject: [PATCH 061/374] min_area size of regions considered for reading order detection passed as an argument for inference --- train/gt_gen_utils.py | 13 +++++++++++-- train/inference.py | 31 ++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 8f72fb8..d3dd7df 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -32,10 +32,16 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m jv = 0 for c in contours: + if len(np.shape(c)) == 3: + c = c[0] + elif len(np.shape(c)) == 2: + pass + #c = c[0] if len(c) < 3: # A polygon cannot have less than 3 points continue - polygon = geometry.Polygon([point[0] for point in c]) + c_e = [point for point in c] + polygon = geometry.Polygon(c_e) # area = cv2.contourArea(c) area = polygon.area # Check that polygon has area greater than minimal area @@ -49,7 +55,10 @@ def filter_contours_area_of_image(image, contours, order_index, max_area, min_ar order_index_filtered = list() #jv = 0 for jv, c in enumerate(contours): - c = c[0] + if len(np.shape(c)) == 3: + c = c[0] + elif len(np.shape(c)) == 2: + pass if len(c) < 3: # A polygon cannot have less than 3 points continue c_e = [point for point in c] diff --git a/train/inference.py b/train/inference.py index 28445e8..c7a8b02 100644 --- a/train/inference.py +++ b/train/inference.py @@ -28,7 +28,7 @@ Tool to load model and predict for given image. """ class sbb_predict: - def __init__(self,image, model, task, config_params_model, patches, save, ground_truth, xml_file, out): + def __init__(self,image, model, task, config_params_model, patches, save, ground_truth, xml_file, out, min_area): self.image=image self.patches=patches self.save=save @@ -38,6 +38,10 @@ class sbb_predict: self.config_params_model=config_params_model self.xml_file = xml_file self.out = out + if min_area: + self.min_area = float(min_area) + else: + self.min_area = 0 def resize_image(self,img_in,input_height,input_width): return cv2.resize( img_in, ( input_width,input_height) ,interpolation=cv2.INTER_NEAREST) @@ -255,11 +259,18 @@ class sbb_predict: ##texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] texts_corr_order_index_int = list(np.array(range(len(co_text_all)))) - min_area = 0 - max_area = 1 + #print(texts_corr_order_index_int) - - ##co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area) + max_area = 1 + #print(np.shape(co_text_all[0]), len( np.shape(co_text_all[0]) ),'co_text_all') + #co_text_all = filter_contours_area_of_image_tables(img_poly, co_text_all, _, max_area, min_area) + #print(co_text_all,'co_text_all') + co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, self.min_area) + + #print(texts_corr_order_index_int) + + #co_text_all = [co_text_all[index] for index in texts_corr_order_index_int] + id_all_text = [id_all_text[index] for index in texts_corr_order_index_int] labels_con = np.zeros((y_len,x_len,len(co_text_all)),dtype='uint8') for i in range(len(co_text_all)): @@ -596,7 +607,13 @@ class sbb_predict: "-xml", help="xml file with layout coordinates that reading order detection will be implemented on. The result will be written in the same xml file.", ) -def main(image, model, patches, save, ground_truth, xml_file, out): + +@click.option( + "--min_area", + "-min", + help="min area size of regions considered for reading order detection. The default value is zero and means that all text regions are considered for reading order.", +) +def main(image, model, patches, save, ground_truth, xml_file, out, min_area): with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) task = config_params_model['task'] @@ -604,7 +621,7 @@ def main(image, model, patches, save, ground_truth, xml_file, out): if not save: print("Error: You used one of segmentation or binarization task but not set -s, you need a filename to save visualized output with -s") sys.exit(1) - x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, xml_file, out) + x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, xml_file, out, min_area) x.run() if __name__=="__main__": From 47a16464518f32427d7ff609bbc572303c2ed148 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 30 May 2024 12:56:56 +0200 Subject: [PATCH 062/374] modifying xml parsing --- train/gt_gen_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index d3dd7df..debaf15 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -122,7 +122,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ ## to do: add footnote to text regions for index in tqdm(range(len(gt_list))): #try: - tree1 = ET.parse(dir_in+'/'+gt_list[index]) + tree1 = ET.parse(dir_in+'/'+gt_list[index], parser = ET.XMLParser(encoding = 'iso-8859-5')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' @@ -658,7 +658,7 @@ def find_new_features_of_contours(contours_main): return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin def read_xml(xml_file): file_name = Path(xml_file).stem - tree1 = ET.parse(xml_file) + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' From 3ef0dbdd4281bfe4cabd13765fc9723ea1e506c2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 30 May 2024 16:59:50 +0200 Subject: [PATCH 063/374] scaling and cropping of labels and org images --- train/custom_config_page2label.json | 5 +- train/generate_gt_for_training.py | 34 ++++++-- train/gt_gen_utils.py | 125 ++++++++++++++++++++++++++-- 3 files changed, 145 insertions(+), 19 deletions(-) diff --git a/train/custom_config_page2label.json b/train/custom_config_page2label.json index e4c02cb..9116ce3 100644 --- a/train/custom_config_page2label.json +++ b/train/custom_config_page2label.json @@ -1,9 +1,8 @@ { -"use_case": "layout", +"use_case": "textline", "textregions":{ "rest_as_paragraph": 1, "header":2 , "heading":2 , "marginalia":3 }, "imageregion":4, "separatorregion":5, "graphicregions" :{"rest_as_decoration":6}, -"artificial_class_on_boundry": ["paragraph"], -"artificial_class_label":7 +"columns_width":{"1":1000, "2":1300, "3":1600, "4":2000, "5":2300, "6":2500} } diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index cf2b2a6..752090c 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -14,10 +14,22 @@ def main(): help="directory of GT page-xml files", type=click.Path(exists=True, file_okay=False), ) +@click.option( + "--dir_images", + "-di", + help="directory of org images. If print space cropping or scaling is needed for labels it would be great to provide the original images to apply the same function on them. So if -ps is not set true or in config files no columns_width key is given this argumnet can be ignored. File stems in this directory should be the same as those in dir_xml.", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--dir_out_images", + "-doi", + help="directory where the output org images after undergoing a process (like print space cropping or scaling) will be written.", + type=click.Path(exists=True, file_okay=False), +) @click.option( "--dir_out", "-do", - help="directory where ground truth images would be written", + help="directory where ground truth label images would be written", type=click.Path(exists=True, file_okay=False), ) @@ -33,8 +45,14 @@ def main(): "-to", help="this defines how output should be. A 2d image array or a 3d image array encoded with RGB color. Just pass 2d or 3d. The file will be saved one directory up. 2D image array is 3d but only information of one channel would be enough since all channels have the same values.", ) +@click.option( + "--printspace", + "-ps", + is_flag=True, + help="if this parameter set to true, generated labels and in the case of provided org images cropping will be imposed and cropped labels and images will be written in output directories.", +) -def pagexml2label(dir_xml,dir_out,type_output,config): +def pagexml2label(dir_xml,dir_out,type_output,config, printspace, dir_images, dir_out_images): if config: with open(config) as f: config_params = json.load(f) @@ -42,7 +60,7 @@ def pagexml2label(dir_xml,dir_out,type_output,config): print("passed") config_params = None gt_list = get_content_of_dir(dir_xml) - get_images_of_ground_truth(gt_list,dir_xml,dir_out,type_output, config, config_params) + get_images_of_ground_truth(gt_list,dir_xml,dir_out,type_output, config, config_params, printspace, dir_images, dir_out_images) @main.command() @click.option( @@ -181,7 +199,7 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i for i in range(len(texts_corr_order_index_int)): for j in range(len(texts_corr_order_index_int)): if i!=j: - input_matrix = np.zeros((input_height,input_width,3)).astype(np.int8) + input_multi_visual_modal = np.zeros((input_height,input_width,3)).astype(np.int8) final_f_name = f_name+'_'+str(indexer+indexer_start) order_class_condition = texts_corr_order_index_int[i]-texts_corr_order_index_int[j] if order_class_condition<0: @@ -189,13 +207,13 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i else: class_type = 0 - input_matrix[:,:,0] = resize_image(labels_con[:,:,i], input_height, input_width) - input_matrix[:,:,1] = resize_image(img_poly[:,:,0], input_height, input_width) - input_matrix[:,:,2] = resize_image(labels_con[:,:,j], input_height, input_width) + input_multi_visual_modal[:,:,0] = resize_image(labels_con[:,:,i], input_height, input_width) + input_multi_visual_modal[:,:,1] = resize_image(img_poly[:,:,0], input_height, input_width) + input_multi_visual_modal[:,:,2] = resize_image(labels_con[:,:,j], input_height, input_width) np.save(os.path.join(dir_out_classes,final_f_name+'.npy' ), class_type) - cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'.png' ), input_matrix) + cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'.png' ), input_multi_visual_modal) indexer = indexer+1 diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index debaf15..d3e95e8 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -115,11 +115,15 @@ def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y img_boundary[:,:][boundary[:,:]==1] =1 return co_text_eroded, img_boundary -def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_file, config_params): +def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_file, config_params, printspace, dir_images, dir_out_images): """ Reading the page xml files and write the ground truth images into given output directory. """ ## to do: add footnote to text regions + + if dir_images: + ls_org_imgs = os.listdir(dir_images) + ls_org_imgs_stem = [item.split('.')[0] for item in ls_org_imgs] for index in tqdm(range(len(gt_list))): #try: tree1 = ET.parse(dir_in+'/'+gt_list[index], parser = ET.XMLParser(encoding = 'iso-8859-5')) @@ -133,6 +137,72 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ y_len=int(jj.attrib['imageHeight']) x_len=int(jj.attrib['imageWidth']) + if 'columns_width' in list(config_params.keys()): + columns_width_dict = config_params['columns_width'] + metadata_element = root1.find(link+'Metadata') + comment_is_sub_element = False + for child in metadata_element: + tag2 = child.tag + if tag2.endswith('}Comments') or tag2.endswith('}comments'): + text_comments = child.text + num_col = int(text_comments.split('num_col')[1]) + comment_is_sub_element = True + if not comment_is_sub_element: + num_col = None + + if num_col: + x_new = columns_width_dict[str(num_col)] + y_new = int ( x_new * (y_len / float(x_len)) ) + + if printspace: + region_tags = np.unique([x for x in alltags if x.endswith('PrintSpace') or x.endswith('Border')]) + co_use_case = [] + + for tag in region_tags: + tag_endings = ['}PrintSpace','}Border'] + + if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + # check the format of coords + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_use_case.append(np.array(c_t_in)) + + img = np.zeros((y_len, x_len, 3)) + + img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) + + img_poly = img_poly.astype(np.uint8) + + imgray = cv2.cvtColor(img_poly, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) + + cnt = contours[np.argmax(cnt_size)] + + x, y, w, h = cv2.boundingRect(cnt) + bb_xywh = [x, y, w, h] + + if config_file and (config_params['use_case']=='textline' or config_params['use_case']=='word' or config_params['use_case']=='glyph' or config_params['use_case']=='printspace'): keys = list(config_params.keys()) if "artificial_class_label" in keys: @@ -186,7 +256,6 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ co_use_case.append(np.array(c_t_in)) - if "artificial_class_label" in keys: img_boundary = np.zeros((y_len, x_len)) erosion_rate = 1 @@ -205,12 +274,32 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + + + if printspace and config_params['use_case']!='printspace': + img_poly = img_poly[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + + if 'columns_width' in list(config_params.keys()) and num_col and config_params['use_case']!='printspace': + img_poly = resize_image(img_poly, y_new, x_new) try: - cv2.imwrite(output_dir + '/' + gt_list[index].split('-')[1].split('.')[0] + '.png', - img_poly) + xml_file_stem = gt_list[index].split('-')[1].split('.')[0] + cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) except: - cv2.imwrite(output_dir + '/' + gt_list[index].split('.')[0] + '.png', img_poly) + xml_file_stem = gt_list[index].split('.')[0] + cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) + + if dir_images: + org_image_name = ls_org_imgs[ls_org_imgs_stem.index(xml_file_stem)] + img_org = cv2.imread(os.path.join(dir_images, org_image_name)) + + if printspace and config_params['use_case']!='printspace': + img_org = img_org[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + + if 'columns_width' in list(config_params.keys()) and num_col and config_params['use_case']!='printspace': + img_org = resize_image(img_org, y_new, x_new) + + cv2.imwrite(os.path.join(dir_out_images, org_image_name), img_org) if config_file and config_params['use_case']=='layout': @@ -616,11 +705,31 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ + if printspace: + img_poly = img_poly[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] - try: - cv2.imwrite(output_dir+'/'+gt_list[index].split('-')[1].split('.')[0]+'.png',img_poly ) + if 'columns_width' in list(config_params.keys()) and num_col: + img_poly = resize_image(img_poly, y_new, x_new) + + try: + xml_file_stem = gt_list[index].split('-')[1].split('.')[0] + cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) except: - cv2.imwrite(output_dir+'/'+gt_list[index].split('.')[0]+'.png',img_poly ) + xml_file_stem = gt_list[index].split('.')[0] + cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) + + + if dir_images: + org_image_name = ls_org_imgs[ls_org_imgs_stem.index(xml_file_stem)] + img_org = cv2.imread(os.path.join(dir_images, org_image_name)) + + if printspace: + img_org = img_org[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + + if 'columns_width' in list(config_params.keys()) and num_col: + img_org = resize_image(img_org, y_new, x_new) + + cv2.imwrite(os.path.join(dir_out_images, org_image_name), img_org) From 13ebe71d1349d5802d9ff5aa1e79e95141185371 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 6 Jun 2024 14:38:29 +0200 Subject: [PATCH 064/374] replacement in a list done correctly --- train/gt_gen_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index d3e95e8..38e77e8 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -636,7 +636,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if 'textregions' in keys: if 'rest_as_paragraph' in types_text: - types_text[types_text=='rest_as_paragraph'] = 'paragraph' + types_text = ['paragraph'if ttind=='rest_as_paragraph' else ttind for ttind in types_text] for element_text in types_text: if element_text == 'paragraph': color_label = labels_rgb_color[ config_params['textregions']['rest_as_paragraph']] @@ -688,7 +688,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if 'textregions' in keys: if 'rest_as_paragraph' in types_text: - types_text[types_text=='rest_as_paragraph'] = 'paragraph' + types_text = ['paragraph'if ttind=='rest_as_paragraph' else ttind for ttind in types_text] for element_text in types_text: if element_text == 'paragraph': color_label = config_params['textregions']['rest_as_paragraph'] From 742e3c2aa28171cbeff8517cf49ab779d196ee23 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 6 Jun 2024 14:46:06 +0200 Subject: [PATCH 065/374] Update README.md --- train/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/train/README.md b/train/README.md index 899c9a3..b9e70a8 100644 --- a/train/README.md +++ b/train/README.md @@ -73,3 +73,6 @@ The output folder should be an empty folder where the output model will be writt * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` * data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. + +#### Additional documentation +Please check the [wiki](https://github.com/qurator-spk/sbb_pixelwise_segmentation/wiki). From 5a5914e06c1185f24de378dc752892e699c0446b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 6 Jun 2024 18:45:47 +0200 Subject: [PATCH 066/374] just defined textregion types can be extracted as label --- train/gt_gen_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 38e77e8..86eb0a1 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -325,6 +325,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ region_tags=np.unique([x for x in alltags if x.endswith('Region')]) co_text = {'drop-capital':[], "footnote":[], "footnote-continued":[], "heading":[], "signature-mark":[], "header":[], "catch-word":[], "page-number":[], "marginalia":[], "paragraph":[]} + all_defined_textregion_types = list(co_text.keys()) co_graphic = {"handwritten-annotation":[], "decoration":[], "stamp":[], "signature":[]} co_sep=[] co_img=[] @@ -359,7 +360,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ else: if "type" in nn.attrib: - c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + if nn.attrib['type'] in all_defined_textregion_types: + c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) break else: @@ -384,8 +386,9 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ else: if "type" in nn.attrib: - c_t_in[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) - sumi+=1 + if nn.attrib['type'] in all_defined_textregion_types: + c_t_in[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 elif vv.tag!=link+'Point' and sumi>=1: From 4c376289e97890a55755e72198d20fde37dd1146 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 6 Jun 2024 18:55:22 +0200 Subject: [PATCH 067/374] just defined graphic region types can be extracted as label --- train/gt_gen_utils.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 86eb0a1..c2360fc 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -327,6 +327,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ co_text = {'drop-capital':[], "footnote":[], "footnote-continued":[], "heading":[], "signature-mark":[], "header":[], "catch-word":[], "page-number":[], "marginalia":[], "paragraph":[]} all_defined_textregion_types = list(co_text.keys()) co_graphic = {"handwritten-annotation":[], "decoration":[], "stamp":[], "signature":[]} + all_defined_graphic_types = list(co_graphic.keys()) co_sep=[] co_img=[] co_table=[] @@ -425,7 +426,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ else: if "type" in nn.attrib: - c_t_in_graphic[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + if nn.attrib['type'] in all_defined_graphic_types: + c_t_in_graphic[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) break else: @@ -450,8 +452,9 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ else: if "type" in nn.attrib: - c_t_in_graphic[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) - sumi+=1 + if nn.attrib['type'] in all_defined_graphic_types: + c_t_in_graphic[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 elif vv.tag!=link+'Point' and sumi>=1: break From cc91e4b12c42076f76bf3e8409c050ad80e9cf78 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 7 Jun 2024 16:24:31 +0200 Subject: [PATCH 068/374] updating train.py --- train/train.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/train/train.py b/train/train.py index f338c78..e16745f 100644 --- a/train/train.py +++ b/train/train.py @@ -59,6 +59,8 @@ def config_params(): pretraining = False # Set to true to load pretrained weights of ResNet50 encoder. scaling_bluring = False # If true, a combination of scaling and blurring will be applied to the image. scaling_binarization = False # If true, a combination of scaling and binarization will be applied to the image. + rotation = False # If true, a 90 degree rotation will be implemeneted. + rotation_not_90 = False # If true rotation based on provided angles with thetha will be implemeneted. scaling_brightness = False # If true, a combination of scaling and brightening will be applied to the image. scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image. thetha = None # Rotate image by these angles for augmentation. From 1921e6754f7abbafb5f7f2731f2d29588bf4eac6 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 10 Jun 2024 22:15:30 +0200 Subject: [PATCH 069/374] updating train.py nontransformer backend --- train/models.py | 13 +++++++++---- train/train.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/train/models.py b/train/models.py index d852ac3..b8b0d27 100644 --- a/train/models.py +++ b/train/models.py @@ -30,8 +30,8 @@ class Patches(layers.Layer): self.patch_size = patch_size def call(self, images): - print(tf.shape(images)[1],'images') - print(self.patch_size,'self.patch_size') + #print(tf.shape(images)[1],'images') + #print(self.patch_size,'self.patch_size') batch_size = tf.shape(images)[0] patches = tf.image.extract_patches( images=images, @@ -41,7 +41,7 @@ class Patches(layers.Layer): padding="VALID", ) patch_dims = patches.shape[-1] - print(patches.shape,patch_dims,'patch_dims') + #print(patches.shape,patch_dims,'patch_dims') patches = tf.reshape(patches, [batch_size, -1, patch_dims]) return patches def get_config(self): @@ -51,6 +51,7 @@ class Patches(layers.Layer): 'patch_size': self.patch_size, }) return config + class PatchEncoder(layers.Layer): def __init__(self, num_patches, projection_dim): @@ -408,7 +409,11 @@ def vit_resnet50_unet(n_classes, patch_size, num_patches, input_height=224, inpu if pretraining: model = Model(inputs, x).load_weights(resnet50_Weights_path) - num_patches = x.shape[1]*x.shape[2] + #num_patches = x.shape[1]*x.shape[2] + + #patch_size_y = input_height / x.shape[1] + #patch_size_x = input_width / x.shape[2] + #patch_size = patch_size_x * patch_size_y patches = Patches(patch_size)(x) # Encode patches. encoded_patches = PatchEncoder(num_patches, projection_dim)(patches) diff --git a/train/train.py b/train/train.py index e16745f..84c9d3b 100644 --- a/train/train.py +++ b/train/train.py @@ -97,8 +97,6 @@ def run(_config, n_classes, n_epochs, input_height, pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name): if task == "segmentation" or task == "enhancement": - - num_patches = transformer_num_patches_xy[0]*transformer_num_patches_xy[1] if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') dir_eval_flowing = os.path.join(dir_output, 'eval') @@ -213,7 +211,15 @@ def run(_config, n_classes, n_epochs, input_height, index_start = 0 if backbone_type=='nontransformer': model = resnet50_unet(n_classes, input_height, input_width, task, weight_decay, pretraining) - elif backbone_type=='nontransformer': + elif backbone_type=='transformer': + num_patches = transformer_num_patches_xy[0]*transformer_num_patches_xy[1] + + if not (num_patches == (input_width / 32) * (input_height / 32)): + print("Error: transformer num patches error. Parameter transformer_num_patches_xy should be set to (input_width/32) = {} and (input_height/32) = {}".format(int(input_width / 32), int(input_height / 32)) ) + sys.exit(1) + if not (transformer_patchsize == 1): + print("Error: transformer patchsize error. Parameter transformer_patchsizeshould set to 1" ) + sys.exit(1) model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width, task, weight_decay, pretraining) #if you want to see the model structure just uncomment model summary. From 29da23da7663ade94f9dc158ba9cd04a39a6f114 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 11 Jun 2024 17:48:30 +0200 Subject: [PATCH 070/374] binarization as a separate task of segmentation --- train/train.py | 13 +++++++------ train/utils.py | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/train/train.py b/train/train.py index 84c9d3b..9e06a66 100644 --- a/train/train.py +++ b/train/train.py @@ -96,7 +96,7 @@ def run(_config, n_classes, n_epochs, input_height, transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name): - if task == "segmentation" or task == "enhancement": + if task == "segmentation" or task == "enhancement" or task == "binarization": if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') dir_eval_flowing = os.path.join(dir_output, 'eval') @@ -194,16 +194,16 @@ def run(_config, n_classes, n_epochs, input_height, if continue_training: if backbone_type=='nontransformer': - if is_loss_soft_dice and task == "segmentation": + if is_loss_soft_dice and (task == "segmentation" or task == "binarization"): model = load_model(dir_of_start_model, compile=True, custom_objects={'soft_dice_loss': soft_dice_loss}) - if weighted_loss and task == "segmentation": + if weighted_loss and (task == "segmentation" or task == "binarization"): model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) if not is_loss_soft_dice and not weighted_loss: model = load_model(dir_of_start_model , compile=True) elif backbone_type=='transformer': - if is_loss_soft_dice and task == "segmentation": + if is_loss_soft_dice and (task == "segmentation" or task == "binarization"): model = load_model(dir_of_start_model, compile=True, custom_objects={"PatchEncoder": PatchEncoder, "Patches": Patches,'soft_dice_loss': soft_dice_loss}) - if weighted_loss and task == "segmentation": + if weighted_loss and (task == "segmentation" or task == "binarization"): model = load_model(dir_of_start_model, compile=True, custom_objects={'loss': weighted_categorical_crossentropy(weights)}) if not is_loss_soft_dice and not weighted_loss: model = load_model(dir_of_start_model , compile=True,custom_objects = {"PatchEncoder": PatchEncoder, "Patches": Patches}) @@ -224,8 +224,9 @@ def run(_config, n_classes, n_epochs, input_height, #if you want to see the model structure just uncomment model summary. #model.summary() + - if task == "segmentation": + if (task == "segmentation" or task == "binarization"): if not is_loss_soft_dice and not weighted_loss: model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=learning_rate), metrics=['accuracy']) diff --git a/train/utils.py b/train/utils.py index a2e8a9c..605d8d1 100644 --- a/train/utils.py +++ b/train/utils.py @@ -309,7 +309,7 @@ def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_c interpolation=cv2.INTER_NEAREST) # Read an image from folder and resize img[i - c] = train_img # add to array - img[0], img[1], and so on. - if task == "segmentation": + if task == "segmentation" or task=="binarization": train_mask = cv2.imread(mask_folder + '/' + filename + '.png') train_mask = get_one_hot(resize_image(train_mask, input_height, input_width), input_height, input_width, n_classes) @@ -569,7 +569,7 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow indexer = 0 for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): img_name = im.split('.')[0] - if task == "segmentation": + if task == "segmentation" or task == "binarization": dir_of_label_file = os.path.join(dir_seg, img_name + '.png') elif task=="enhancement": dir_of_label_file = os.path.join(dir_seg, im) From 95faf1a4c8bc25ffe6d89fa2d296fccf95479e18 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 12 Jun 2024 13:26:27 +0200 Subject: [PATCH 071/374] transformer patch size is dynamic now. --- train/config_params.json | 28 +++++++++++++----------- train/models.py | 47 ++++++++++++++++++++++++++++++++-------- train/train.py | 30 ++++++++++++++++++------- 3 files changed, 75 insertions(+), 30 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index 8a56de5..6b8b6ed 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,42 +1,44 @@ { - "backbone_type" : "nontransformer", - "task": "classification", + "backbone_type" : "transformer", + "task": "binarization", "n_classes" : 2, - "n_epochs" : 20, - "input_height" : 448, - "input_width" : 448, + "n_epochs" : 1, + "input_height" : 224, + "input_width" : 672, "weight_decay" : 1e-6, - "n_batch" : 6, + "n_batch" : 1, "learning_rate": 1e-4, - "f1_threshold_classification": 0.8, "patches" : true, "pretraining" : true, "augmentation" : false, "flip_aug" : false, "blur_aug" : false, "scaling" : true, + "degrading": false, + "brightening": false, "binarization" : false, "scaling_bluring" : false, "scaling_binarization" : false, "scaling_flip" : false, "rotation": false, "rotation_not_90": false, - "transformer_num_patches_xy": [28, 28], - "transformer_patchsize": 1, + "transformer_num_patches_xy": [7, 7], + "transformer_patchsize_x": 3, + "transformer_patchsize_y": 1, + "transformer_projection_dim": 192, "blur_k" : ["blur","guass","median"], "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], "brightness" : [1.3, 1.5, 1.7, 2], "degrade_scales" : [0.2, 0.4], "flip_index" : [0, 1, -1], "thetha" : [10, -10], - "classification_classes_name" : {"0":"apple", "1":"orange"}, "continue_training": false, "index_start" : 0, "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" + "dir_train": "/home/vahid/Documents/test/training_data_sample_binarization", + "dir_eval": "/home/vahid/Documents/test/eval", + "dir_output": "/home/vahid/Documents/test/out" } diff --git a/train/models.py b/train/models.py index b8b0d27..1abf304 100644 --- a/train/models.py +++ b/train/models.py @@ -6,25 +6,49 @@ from tensorflow.keras import layers from tensorflow.keras.regularizers import l2 mlp_head_units = [2048, 1024] -projection_dim = 64 +#projection_dim = 64 transformer_layers = 8 num_heads = 4 resnet50_Weights_path = './pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' IMAGE_ORDERING = 'channels_last' MERGE_AXIS = -1 -transformer_units = [ - projection_dim * 2, - projection_dim, -] # Size of the transformer layers def mlp(x, hidden_units, dropout_rate): for units in hidden_units: x = layers.Dense(units, activation=tf.nn.gelu)(x) x = layers.Dropout(dropout_rate)(x) return x - class Patches(layers.Layer): + def __init__(self, patch_size_x, patch_size_y):#__init__(self, **kwargs):#:__init__(self, patch_size):#__init__(self, **kwargs): + super(Patches, self).__init__() + self.patch_size_x = patch_size_x + self.patch_size_y = patch_size_y + + def call(self, images): + #print(tf.shape(images)[1],'images') + #print(self.patch_size,'self.patch_size') + batch_size = tf.shape(images)[0] + patches = tf.image.extract_patches( + images=images, + sizes=[1, self.patch_size_y, self.patch_size_x, 1], + strides=[1, self.patch_size_y, self.patch_size_x, 1], + rates=[1, 1, 1, 1], + padding="VALID", + ) + patch_dims = patches.shape[-1] + patches = tf.reshape(patches, [batch_size, -1, patch_dims]) + return patches + def get_config(self): + + config = super().get_config().copy() + config.update({ + 'patch_size_x': self.patch_size_x, + 'patch_size_y': self.patch_size_y, + }) + return config + +class Patches_old(layers.Layer): def __init__(self, patch_size):#__init__(self, **kwargs):#:__init__(self, patch_size):#__init__(self, **kwargs): super(Patches, self).__init__() self.patch_size = patch_size @@ -369,8 +393,13 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, task="segmentati return model -def vit_resnet50_unet(n_classes, patch_size, num_patches, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): +def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): inputs = layers.Input(shape=(input_height, input_width, 3)) + + transformer_units = [ + projection_dim * 2, + projection_dim, + ] # Size of the transformer layers IMAGE_ORDERING = 'channels_last' bn_axis=3 @@ -414,7 +443,7 @@ def vit_resnet50_unet(n_classes, patch_size, num_patches, input_height=224, inpu #patch_size_y = input_height / x.shape[1] #patch_size_x = input_width / x.shape[2] #patch_size = patch_size_x * patch_size_y - patches = Patches(patch_size)(x) + patches = Patches(patch_size_x, patch_size_y)(x) # Encode patches. encoded_patches = PatchEncoder(num_patches, projection_dim)(patches) @@ -434,7 +463,7 @@ def vit_resnet50_unet(n_classes, patch_size, num_patches, input_height=224, inpu # Skip connection 2. encoded_patches = layers.Add()([x3, x2]) - encoded_patches = tf.reshape(encoded_patches, [-1, x.shape[1], x.shape[2], 64]) + encoded_patches = tf.reshape(encoded_patches, [-1, x.shape[1], x.shape[2] , int( projection_dim / (patch_size_x * patch_size_y) )]) v1024_2048 = Conv2D( 1024 , (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(encoded_patches) v1024_2048 = (BatchNormalization(axis=bn_axis))(v1024_2048) diff --git a/train/train.py b/train/train.py index 9e06a66..bafcc9e 100644 --- a/train/train.py +++ b/train/train.py @@ -70,8 +70,10 @@ def config_params(): brightness = None # Brighten image for augmentation. flip_index = None # Flip image for augmentation. continue_training = False # Set to true if you would like to continue training an already trained a model. - transformer_patchsize = None # Patch size of vision transformer patches. + transformer_patchsize_x = None # Patch size of vision transformer patches. + transformer_patchsize_y = None transformer_num_patches_xy = None # Number of patches for vision transformer. + transformer_projection_dim = 64 # Transformer projection dimension index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. @@ -92,7 +94,7 @@ def run(_config, n_classes, n_epochs, input_height, brightening, binarization, blur_k, scales, degrade_scales, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, - thetha, scaling_flip, continue_training, transformer_patchsize, + thetha, scaling_flip, continue_training, transformer_projection_dim, transformer_patchsize_x, transformer_patchsize_y, transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name): @@ -212,15 +214,27 @@ def run(_config, n_classes, n_epochs, input_height, if backbone_type=='nontransformer': model = resnet50_unet(n_classes, input_height, input_width, task, weight_decay, pretraining) elif backbone_type=='transformer': - num_patches = transformer_num_patches_xy[0]*transformer_num_patches_xy[1] + num_patches_x = transformer_num_patches_xy[0] + num_patches_y = transformer_num_patches_xy[1] + num_patches = num_patches_x * num_patches_y - if not (num_patches == (input_width / 32) * (input_height / 32)): - print("Error: transformer num patches error. Parameter transformer_num_patches_xy should be set to (input_width/32) = {} and (input_height/32) = {}".format(int(input_width / 32), int(input_height / 32)) ) + ##if not (num_patches == (input_width / 32) * (input_height / 32)): + ##print("Error: transformer num patches error. Parameter transformer_num_patches_xy should be set to (input_width/32) = {} and (input_height/32) = {}".format(int(input_width / 32), int(input_height / 32)) ) + ##sys.exit(1) + #if not (transformer_patchsize == 1): + #print("Error: transformer patchsize error. Parameter transformer_patchsizeshould set to 1" ) + #sys.exit(1) + if (input_height != (num_patches_y * transformer_patchsize_y * 32) ): + print("Error: transformer_patchsize_y or transformer_num_patches_xy height value error . input_height should be equal to ( transformer_num_patches_xy height value * transformer_patchsize_y * 32)") sys.exit(1) - if not (transformer_patchsize == 1): - print("Error: transformer patchsize error. Parameter transformer_patchsizeshould set to 1" ) + if (input_width != (num_patches_x * transformer_patchsize_x * 32) ): + print("Error: transformer_patchsize_x or transformer_num_patches_xy width value error . input_width should be equal to ( transformer_num_patches_xy width value * transformer_patchsize_x * 32)") sys.exit(1) - model = vit_resnet50_unet(n_classes, transformer_patchsize, num_patches, input_height, input_width, task, weight_decay, pretraining) + if (transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x)) != 0: + print("Error: transformer_projection_dim error. The remainder when parameter transformer_projection_dim is divided by (transformer_patchsize_y*transformer_patchsize_x) should be zero") + sys.exit(1) + + model = vit_resnet50_unet(n_classes, transformer_patchsize_x, transformer_patchsize_y, num_patches, transformer_projection_dim, input_height, input_width, task, weight_decay, pretraining) #if you want to see the model structure just uncomment model summary. #model.summary() From 22d7359db2b1660272a32dd2e43f69f67373883f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 12 Jun 2024 17:39:57 +0200 Subject: [PATCH 072/374] Transformer+CNN structure is added to vision transformer type --- train/config_params.json | 16 +++-- train/models.py | 142 ++++++++++++++++++++++++++++++++++++--- train/train.py | 57 ++++++++++------ 3 files changed, 176 insertions(+), 39 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index 6b8b6ed..d72530e 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -2,9 +2,9 @@ "backbone_type" : "transformer", "task": "binarization", "n_classes" : 2, - "n_epochs" : 1, + "n_epochs" : 2, "input_height" : 224, - "input_width" : 672, + "input_width" : 224, "weight_decay" : 1e-6, "n_batch" : 1, "learning_rate": 1e-4, @@ -22,10 +22,14 @@ "scaling_flip" : false, "rotation": false, "rotation_not_90": false, - "transformer_num_patches_xy": [7, 7], - "transformer_patchsize_x": 3, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 192, + "transformer_num_patches_xy": [56, 56], + "transformer_patchsize_x": 4, + "transformer_patchsize_y": 4, + "transformer_projection_dim": 64, + "transformer_mlp_head_units": [128, 64], + "transformer_layers": 1, + "transformer_num_heads": 1, + "transformer_cnn_first": false, "blur_k" : ["blur","guass","median"], "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], "brightness" : [1.3, 1.5, 1.7, 2], diff --git a/train/models.py b/train/models.py index 1abf304..8841bd3 100644 --- a/train/models.py +++ b/train/models.py @@ -5,10 +5,10 @@ from tensorflow.keras.layers import * from tensorflow.keras import layers from tensorflow.keras.regularizers import l2 -mlp_head_units = [2048, 1024] -#projection_dim = 64 -transformer_layers = 8 -num_heads = 4 +##mlp_head_units = [512, 256]#[2048, 1024] +###projection_dim = 64 +##transformer_layers = 2#8 +##num_heads = 1#4 resnet50_Weights_path = './pretrained_model/resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5' IMAGE_ORDERING = 'channels_last' MERGE_AXIS = -1 @@ -36,7 +36,8 @@ class Patches(layers.Layer): rates=[1, 1, 1, 1], padding="VALID", ) - patch_dims = patches.shape[-1] + #patch_dims = patches.shape[-1] + patch_dims = tf.shape(patches)[-1] patches = tf.reshape(patches, [batch_size, -1, patch_dims]) return patches def get_config(self): @@ -393,13 +394,13 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, task="segmentati return model -def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): +def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=[128, 64], transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): inputs = layers.Input(shape=(input_height, input_width, 3)) - transformer_units = [ - projection_dim * 2, - projection_dim, - ] # Size of the transformer layers + #transformer_units = [ + #projection_dim * 2, + #projection_dim, + #] # Size of the transformer layers IMAGE_ORDERING = 'channels_last' bn_axis=3 @@ -459,7 +460,7 @@ def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, projec # Layer normalization 2. x3 = layers.LayerNormalization(epsilon=1e-6)(x2) # MLP. - x3 = mlp(x3, hidden_units=transformer_units, dropout_rate=0.1) + x3 = mlp(x3, hidden_units=mlp_head_units, dropout_rate=0.1) # Skip connection 2. encoded_patches = layers.Add()([x3, x2]) @@ -515,6 +516,125 @@ def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, projec return model +def vit_resnet50_unet_transformer_before_cnn(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=[128, 64], transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): + inputs = layers.Input(shape=(input_height, input_width, 3)) + + ##transformer_units = [ + ##projection_dim * 2, + ##projection_dim, + ##] # Size of the transformer layers + IMAGE_ORDERING = 'channels_last' + bn_axis=3 + + patches = Patches(patch_size_x, patch_size_y)(inputs) + # Encode patches. + encoded_patches = PatchEncoder(num_patches, projection_dim)(patches) + + for _ in range(transformer_layers): + # Layer normalization 1. + x1 = layers.LayerNormalization(epsilon=1e-6)(encoded_patches) + # Create a multi-head attention layer. + attention_output = layers.MultiHeadAttention( + num_heads=num_heads, key_dim=projection_dim, dropout=0.1 + )(x1, x1) + # Skip connection 1. + x2 = layers.Add()([attention_output, encoded_patches]) + # Layer normalization 2. + x3 = layers.LayerNormalization(epsilon=1e-6)(x2) + # MLP. + x3 = mlp(x3, hidden_units=mlp_head_units, dropout_rate=0.1) + # Skip connection 2. + encoded_patches = layers.Add()([x3, x2]) + + encoded_patches = tf.reshape(encoded_patches, [-1, input_height, input_width , int( projection_dim / (patch_size_x * patch_size_y) )]) + + encoded_patches = Conv2D(3, (1, 1), padding='same', data_format=IMAGE_ORDERING, kernel_regularizer=l2(weight_decay), name='convinput')(encoded_patches) + + x = ZeroPadding2D((3, 3), data_format=IMAGE_ORDERING)(encoded_patches) + x = Conv2D(64, (7, 7), data_format=IMAGE_ORDERING, strides=(2, 2),kernel_regularizer=l2(weight_decay), name='conv1')(x) + f1 = x + + x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) + x = Activation('relu')(x) + x = MaxPooling2D((3, 3), data_format=IMAGE_ORDERING, strides=(2, 2))(x) + + x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) + x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') + x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') + f2 = one_side_pad(x) + + x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') + x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') + f3 = x + + x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') + x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') + f4 = x + + x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') + x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') + f5 = x + + if pretraining: + model = Model(encoded_patches, x).load_weights(resnet50_Weights_path) + + v1024_2048 = Conv2D( 1024 , (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(x) + v1024_2048 = (BatchNormalization(axis=bn_axis))(v1024_2048) + v1024_2048 = Activation('relu')(v1024_2048) + + o = (UpSampling2D( (2, 2), data_format=IMAGE_ORDERING))(v1024_2048) + o = (concatenate([o, f4],axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(512, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o ,f3], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(256, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f2], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(128, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, f1], axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(64, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = (UpSampling2D((2, 2), data_format=IMAGE_ORDERING))(o) + o = (concatenate([o, inputs],axis=MERGE_AXIS)) + o = (ZeroPadding2D((1, 1), data_format=IMAGE_ORDERING))(o) + o = (Conv2D(32, (3, 3), padding='valid', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay)))(o) + o = (BatchNormalization(axis=bn_axis))(o) + o = Activation('relu')(o) + + o = Conv2D(n_classes, (1, 1), padding='same', data_format=IMAGE_ORDERING,kernel_regularizer=l2(weight_decay))(o) + if task == "segmentation": + o = (BatchNormalization(axis=bn_axis))(o) + o = (Activation('softmax'))(o) + else: + o = (Activation('sigmoid'))(o) + + model = Model(inputs=inputs, outputs=o) + + return model + def resnet50_classifier(n_classes,input_height=224,input_width=224,weight_decay=1e-6,pretraining=False): include_top=True assert input_height%32 == 0 diff --git a/train/train.py b/train/train.py index bafcc9e..71f31f3 100644 --- a/train/train.py +++ b/train/train.py @@ -70,10 +70,14 @@ def config_params(): brightness = None # Brighten image for augmentation. flip_index = None # Flip image for augmentation. continue_training = False # Set to true if you would like to continue training an already trained a model. - transformer_patchsize_x = None # Patch size of vision transformer patches. - transformer_patchsize_y = None - transformer_num_patches_xy = None # Number of patches for vision transformer. - transformer_projection_dim = 64 # Transformer projection dimension + transformer_patchsize_x = None # Patch size of vision transformer patches in x direction. + transformer_patchsize_y = None # Patch size of vision transformer patches in y direction. + transformer_num_patches_xy = None # Number of patches for vision transformer in x and y direction respectively. + transformer_projection_dim = 64 # Transformer projection dimension. Default value is 64. + transformer_mlp_head_units = [128, 64] # Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64] + transformer_layers = 8 # transformer layers. Default value is 8. + transformer_num_heads = 4 # Transformer number of heads. Default value is 4. + transformer_cnn_first = True # We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. index_start = 0 # Index of model to continue training from. E.g. if you trained for 3 epochs and last index is 2, to continue from model_1.h5, set "index_start" to 3 to start naming model with index 3. dir_of_start_model = '' # Directory containing pretrained encoder to continue training the model. is_loss_soft_dice = False # Use soft dice as loss function. When set to true, "weighted_loss" must be false. @@ -94,7 +98,9 @@ def run(_config, n_classes, n_epochs, input_height, brightening, binarization, blur_k, scales, degrade_scales, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, - thetha, scaling_flip, continue_training, transformer_projection_dim, transformer_patchsize_x, transformer_patchsize_y, + thetha, scaling_flip, continue_training, transformer_projection_dim, + transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_cnn_first, + transformer_patchsize_x, transformer_patchsize_y, transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name): @@ -218,26 +224,33 @@ def run(_config, n_classes, n_epochs, input_height, num_patches_y = transformer_num_patches_xy[1] num_patches = num_patches_x * num_patches_y - ##if not (num_patches == (input_width / 32) * (input_height / 32)): - ##print("Error: transformer num patches error. Parameter transformer_num_patches_xy should be set to (input_width/32) = {} and (input_height/32) = {}".format(int(input_width / 32), int(input_height / 32)) ) - ##sys.exit(1) - #if not (transformer_patchsize == 1): - #print("Error: transformer patchsize error. Parameter transformer_patchsizeshould set to 1" ) - #sys.exit(1) - if (input_height != (num_patches_y * transformer_patchsize_y * 32) ): - print("Error: transformer_patchsize_y or transformer_num_patches_xy height value error . input_height should be equal to ( transformer_num_patches_xy height value * transformer_patchsize_y * 32)") - sys.exit(1) - if (input_width != (num_patches_x * transformer_patchsize_x * 32) ): - print("Error: transformer_patchsize_x or transformer_num_patches_xy width value error . input_width should be equal to ( transformer_num_patches_xy width value * transformer_patchsize_x * 32)") - sys.exit(1) - if (transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x)) != 0: - print("Error: transformer_projection_dim error. The remainder when parameter transformer_projection_dim is divided by (transformer_patchsize_y*transformer_patchsize_x) should be zero") - sys.exit(1) + if transformer_cnn_first: + if (input_height != (num_patches_y * transformer_patchsize_y * 32) ): + print("Error: transformer_patchsize_y or transformer_num_patches_xy height value error . input_height should be equal to ( transformer_num_patches_xy height value * transformer_patchsize_y * 32)") + sys.exit(1) + if (input_width != (num_patches_x * transformer_patchsize_x * 32) ): + print("Error: transformer_patchsize_x or transformer_num_patches_xy width value error . input_width should be equal to ( transformer_num_patches_xy width value * transformer_patchsize_x * 32)") + sys.exit(1) + if (transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x)) != 0: + print("Error: transformer_projection_dim error. The remainder when parameter transformer_projection_dim is divided by (transformer_patchsize_y*transformer_patchsize_x) should be zero") + sys.exit(1) + - model = vit_resnet50_unet(n_classes, transformer_patchsize_x, transformer_patchsize_y, num_patches, transformer_projection_dim, input_height, input_width, task, weight_decay, pretraining) + model = vit_resnet50_unet(n_classes, transformer_patchsize_x, transformer_patchsize_y, num_patches, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_projection_dim, input_height, input_width, task, weight_decay, pretraining) + else: + if (input_height != (num_patches_y * transformer_patchsize_y) ): + print("Error: transformer_patchsize_y or transformer_num_patches_xy height value error . input_height should be equal to ( transformer_num_patches_xy height value * transformer_patchsize_y)") + sys.exit(1) + if (input_width != (num_patches_x * transformer_patchsize_x) ): + print("Error: transformer_patchsize_x or transformer_num_patches_xy width value error . input_width should be equal to ( transformer_num_patches_xy width value * transformer_patchsize_x)") + sys.exit(1) + if (transformer_projection_dim % (transformer_patchsize_y * transformer_patchsize_x)) != 0: + print("Error: transformer_projection_dim error. The remainder when parameter transformer_projection_dim is divided by (transformer_patchsize_y*transformer_patchsize_x) should be zero") + sys.exit(1) + model = vit_resnet50_unet_transformer_before_cnn(n_classes, transformer_patchsize_x, transformer_patchsize_y, num_patches, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_projection_dim, input_height, input_width, task, weight_decay, pretraining) #if you want to see the model structure just uncomment model summary. - #model.summary() + model.summary() if (task == "segmentation" or task == "binarization"): From 66022cf771dafd0cafa0734b545e60fc44fa07af Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 12 Jun 2024 17:40:40 +0200 Subject: [PATCH 073/374] update config --- train/config_params.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index d72530e..a89cbb5 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -42,7 +42,7 @@ "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "/home/vahid/Documents/test/training_data_sample_binarization", - "dir_eval": "/home/vahid/Documents/test/eval", - "dir_output": "/home/vahid/Documents/test/out" + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" } From b3cd01de3761ce251b9171aa8f48318d926594f5 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 21 Jun 2024 13:06:26 +0200 Subject: [PATCH 074/374] update reading order machine based --- train/generate_gt_for_training.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 752090c..cfcc151 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -163,8 +163,7 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i #print('########################') xml_file = os.path.join(dir_xml,ind_xml ) f_name = ind_xml.split('.')[0] - file_name, id_paragraph, id_header,co_text_paragraph,\ - co_text_header,tot_region_ref,x_len, y_len,index_tot_regions,img_poly = read_xml(xml_file) + _, _, _, file_name, id_paragraph, id_header,co_text_paragraph,co_text_header,tot_region_ref,x_len, y_len,index_tot_regions,img_poly = read_xml(xml_file) id_all_text = id_paragraph + id_header co_text_all = co_text_paragraph + co_text_header From fe69b9c4a8428cc6a957f2b40c5aa559dd25416b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 21 Jun 2024 23:42:25 +0200 Subject: [PATCH 075/374] update inference --- train/inference.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/train/inference.py b/train/inference.py index c7a8b02..3fec9c2 100644 --- a/train/inference.py +++ b/train/inference.py @@ -557,6 +557,10 @@ class sbb_predict: res=self.predict() if (self.task == 'classification' or self.task == 'reading_order'): pass + elif self.task == 'enhancement': + if self.save: + print(self.save) + cv2.imwrite(self.save,res) else: img_seg_overlayed = self.visualize_model_output(res, self.img_org, self.task) if self.save: From 9260d2962a0fbdcc30ae836d5e21af2122764aa7 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 9 Jul 2024 03:04:29 +0200 Subject: [PATCH 076/374] resolving typo --- train/gt_gen_utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index c2360fc..c264f4c 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -304,8 +304,9 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if config_file and config_params['use_case']=='layout': keys = list(config_params.keys()) - if "artificial_class_on_boundry" in keys: - elements_with_artificial_class = list(config_params['artificial_class_on_boundry']) + + if "artificial_class_on_boundary" in keys: + elements_with_artificial_class = list(config_params['artificial_class_on_boundary']) artificial_class_rgb_color = (255,255,0) artificial_class_label = config_params['artificial_class_label'] #values = config_params.values() @@ -567,8 +568,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ elif vv.tag!=link+'Point' and sumi>=1: break co_noise.append(np.array(c_t_in)) - - if "artificial_class_on_boundry" in keys: + + if "artificial_class_on_boundary" in keys: img_boundary = np.zeros( (y_len,x_len) ) if "paragraph" in elements_with_artificial_class: erosion_rate = 2 @@ -655,7 +656,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly=cv2.fillPoly(img, pts =co_text[element_text], color=color_label) - if "artificial_class_on_boundry" in keys: + if "artificial_class_on_boundary" in keys: img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] @@ -706,7 +707,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ color_label = config_params['textregions'][element_text] img_poly=cv2.fillPoly(img, pts =co_text[element_text], color=color_label) - if "artificial_class_on_boundry" in keys: + if "artificial_class_on_boundary" in keys: img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label From 3bceec9c19158030acdb59f8f84c2d0d66382414 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 16 Jul 2024 18:29:27 +0200 Subject: [PATCH 077/374] printspace_as_class_in_layout is integrated. Printspace can be defined as a class for layout segmentation --- train/gt_gen_utils.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index c264f4c..1df7b2a 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -154,7 +154,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ x_new = columns_width_dict[str(num_col)] y_new = int ( x_new * (y_len / float(x_len)) ) - if printspace: + if printspace or "printspace_as_class_in_layout" in list(config_params.keys()): region_tags = np.unique([x for x in alltags if x.endswith('PrintSpace') or x.endswith('Border')]) co_use_case = [] @@ -279,6 +279,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if printspace and config_params['use_case']!='printspace': img_poly = img_poly[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2], :] + if 'columns_width' in list(config_params.keys()) and num_col and config_params['use_case']!='printspace': img_poly = resize_image(img_poly, y_new, x_new) @@ -310,6 +311,10 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ artificial_class_rgb_color = (255,255,0) artificial_class_label = config_params['artificial_class_label'] #values = config_params.values() + + if "printspace_as_class_in_layout" in list(config_params.keys()): + printspace_class_rgb_color = (125,125,255) + printspace_class_label = config_params['printspace_as_class_in_layout'] if 'textregions' in keys: types_text_dict = config_params['textregions'] @@ -614,7 +619,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ - img = np.zeros( (y_len,x_len,3) ) + img = np.zeros( (y_len,x_len,3) ) if output_type == '3d': if 'graphicregions' in keys: @@ -661,6 +666,15 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + + if "printspace_as_class_in_layout" in list(config_params.keys()): + printspace_mask = np.zeros((img_poly.shape[0], img_poly.shape[1])) + printspace_mask[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2]] = 1 + + img_poly[:,:,0][printspace_mask[:,:] == 0] = printspace_class_rgb_color[0] + img_poly[:,:,1][printspace_mask[:,:] == 0] = printspace_class_rgb_color[1] + img_poly[:,:,2][printspace_mask[:,:] == 0] = printspace_class_rgb_color[2] + @@ -709,6 +723,14 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "artificial_class_on_boundary" in keys: img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label + + if "printspace_as_class_in_layout" in list(config_params.keys()): + printspace_mask = np.zeros((img_poly.shape[0], img_poly.shape[1])) + printspace_mask[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2]] = 1 + + img_poly[:,:,0][printspace_mask[:,:] == 0] = printspace_class_label + img_poly[:,:,1][printspace_mask[:,:] == 0] = printspace_class_label + img_poly[:,:,2][printspace_mask[:,:] == 0] = printspace_class_label From 453d0fbf9220122096fd4578695783faa35823b7 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 17 Jul 2024 17:14:20 +0200 Subject: [PATCH 078/374] adding degrading and brightness augmentation to no patches case training --- train/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/train/utils.py b/train/utils.py index 605d8d1..7a2274c 100644 --- a/train/utils.py +++ b/train/utils.py @@ -597,6 +597,14 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) indexer += 1 + if brightening: + for factor in brightness: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(do_brightening(dir_img + '/' +im, factor), input_height, input_width))) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + indexer += 1 if binarization: cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', @@ -606,6 +614,15 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) indexer += 1 + if degrading: + for degrade_scale_ind in degrade_scales: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(do_degrading(cv2.imread(dir_img + '/' + im), degrade_scale_ind), input_height, input_width))) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + indexer += 1 + if patches: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, From 861f0b1ebd39d8d2c7d127a0d335f8a3ef17c6e2 Mon Sep 17 00:00:00 2001 From: b-vr103 Date: Wed, 17 Jul 2024 18:20:24 +0200 Subject: [PATCH 079/374] brightness augmentation modified --- train/utils.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/train/utils.py b/train/utils.py index 7a2274c..891ee15 100644 --- a/train/utils.py +++ b/train/utils.py @@ -599,12 +599,15 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow indexer += 1 if brightening: for factor in brightness: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - (resize_image(do_brightening(dir_img + '/' +im, factor), input_height, input_width))) + try: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(do_brightening(dir_img + '/' +im, factor), input_height, input_width))) - cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', - resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) - indexer += 1 + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + indexer += 1 + except: + pass if binarization: cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', From 840d7c2283d6b71e083c6f10bf3b2e4b8f2e9102 Mon Sep 17 00:00:00 2001 From: b-vr103 Date: Tue, 23 Jul 2024 11:29:05 +0200 Subject: [PATCH 080/374] increasing margin in the case of pixelwise inference --- train/inference.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/train/inference.py b/train/inference.py index 3fec9c2..49bebf8 100644 --- a/train/inference.py +++ b/train/inference.py @@ -219,7 +219,7 @@ class sbb_predict: added_image = cv2.addWeighted(img,0.5,output,0.1,0) - return added_image + return added_image, output def predict(self): self.start_new_session_and_model() @@ -444,7 +444,7 @@ class sbb_predict: if img.shape[1] < self.img_width: img = cv2.resize(img, (self.img_height, img.shape[0]), interpolation=cv2.INTER_NEAREST) - margin = int(0 * self.img_width) + margin = int(0.1 * self.img_width) width_mid = self.img_width - 2 * margin height_mid = self.img_height - 2 * margin img = img / float(255.0) @@ -562,9 +562,10 @@ class sbb_predict: print(self.save) cv2.imwrite(self.save,res) else: - img_seg_overlayed = self.visualize_model_output(res, self.img_org, self.task) + img_seg_overlayed, only_prediction = self.visualize_model_output(res, self.img_org, self.task) if self.save: cv2.imwrite(self.save,img_seg_overlayed) + cv2.imwrite('./layout.png', only_prediction) if self.ground_truth: gt_img=cv2.imread(self.ground_truth) From 2c822dae4e49d970d26a7776e20f55f34144d79e Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 24 Jul 2024 16:52:05 +0200 Subject: [PATCH 081/374] erosion and dilation parameters are changed & separators are written in label images after artificial label --- train/gt_gen_utils.py | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 1df7b2a..253c44a 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -577,8 +577,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "artificial_class_on_boundary" in keys: img_boundary = np.zeros( (y_len,x_len) ) if "paragraph" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 + erosion_rate = 0#2 + dilation_rate = 3#4 co_text['paragraph'], img_boundary = update_region_contours(co_text['paragraph'], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "drop-capital" in elements_with_artificial_class: erosion_rate = 0 @@ -586,35 +586,35 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ co_text["drop-capital"], img_boundary = update_region_contours(co_text["drop-capital"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "catch-word" in elements_with_artificial_class: erosion_rate = 0 - dilation_rate = 4 + dilation_rate = 2#4 co_text["catch-word"], img_boundary = update_region_contours(co_text["catch-word"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "page-number" in elements_with_artificial_class: erosion_rate = 0 - dilation_rate = 4 + dilation_rate = 2#4 co_text["page-number"], img_boundary = update_region_contours(co_text["page-number"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "header" in elements_with_artificial_class: - erosion_rate = 1 - dilation_rate = 4 + erosion_rate = 0#1 + dilation_rate = 3#4 co_text["header"], img_boundary = update_region_contours(co_text["header"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "heading" in elements_with_artificial_class: - erosion_rate = 1 - dilation_rate = 4 + erosion_rate = 0#1 + dilation_rate = 3#4 co_text["heading"], img_boundary = update_region_contours(co_text["heading"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "signature-mark" in elements_with_artificial_class: erosion_rate = 1 dilation_rate = 4 co_text["signature-mark"], img_boundary = update_region_contours(co_text["signature-mark"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "marginalia" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 + erosion_rate = 0#2 + dilation_rate = 3#4 co_text["marginalia"], img_boundary = update_region_contours(co_text["marginalia"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "footnote" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 + erosion_rate = 0#2 + dilation_rate = 2#4 co_text["footnote"], img_boundary = update_region_contours(co_text["footnote"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "footnote-continued" in elements_with_artificial_class: - erosion_rate = 2 - dilation_rate = 4 + erosion_rate = 0#2 + dilation_rate = 2#4 co_text["footnote-continued"], img_boundary = update_region_contours(co_text["footnote-continued"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) @@ -639,8 +639,6 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if 'imageregion' in keys: img_poly=cv2.fillPoly(img, pts =co_img, color=labels_rgb_color[ config_params['imageregion']]) - if 'separatorregion' in keys: - img_poly=cv2.fillPoly(img, pts =co_sep, color=labels_rgb_color[ config_params['separatorregion']]) if 'tableregion' in keys: img_poly=cv2.fillPoly(img, pts =co_table, color=labels_rgb_color[ config_params['tableregion']]) if 'noiseregion' in keys: @@ -666,6 +664,9 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + if 'separatorregion' in keys: + img_poly=cv2.fillPoly(img, pts =co_sep, color=labels_rgb_color[ config_params['separatorregion']]) + if "printspace_as_class_in_layout" in list(config_params.keys()): printspace_mask = np.zeros((img_poly.shape[0], img_poly.shape[1])) @@ -697,9 +698,6 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if 'imageregion' in keys: color_label = config_params['imageregion'] img_poly=cv2.fillPoly(img, pts =co_img, color=(color_label,color_label,color_label)) - if 'separatorregion' in keys: - color_label = config_params['separatorregion'] - img_poly=cv2.fillPoly(img, pts =co_sep, color=(color_label,color_label,color_label)) if 'tableregion' in keys: color_label = config_params['tableregion'] img_poly=cv2.fillPoly(img, pts =co_table, color=(color_label,color_label,color_label)) @@ -724,6 +722,10 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "artificial_class_on_boundary" in keys: img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label + if 'separatorregion' in keys: + color_label = config_params['separatorregion'] + img_poly=cv2.fillPoly(img, pts =co_sep, color=(color_label,color_label,color_label)) + if "printspace_as_class_in_layout" in list(config_params.keys()): printspace_mask = np.zeros((img_poly.shape[0], img_poly.shape[1])) printspace_mask[bb_xywh[1]:bb_xywh[1]+bb_xywh[3], bb_xywh[0]:bb_xywh[0]+bb_xywh[2]] = 1 From 6fb28d6ce8cab024595a8a787d92129fbbeaf3c3 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 1 Aug 2024 14:30:51 +0200 Subject: [PATCH 082/374] erosion rate changed --- train/gt_gen_utils.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 253c44a..13010bf 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -577,36 +577,36 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "artificial_class_on_boundary" in keys: img_boundary = np.zeros( (y_len,x_len) ) if "paragraph" in elements_with_artificial_class: - erosion_rate = 0#2 - dilation_rate = 3#4 + erosion_rate = 2 + dilation_rate = 4 co_text['paragraph'], img_boundary = update_region_contours(co_text['paragraph'], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "drop-capital" in elements_with_artificial_class: - erosion_rate = 0 - dilation_rate = 4 + erosion_rate = 1 + dilation_rate = 3 co_text["drop-capital"], img_boundary = update_region_contours(co_text["drop-capital"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "catch-word" in elements_with_artificial_class: erosion_rate = 0 - dilation_rate = 2#4 + dilation_rate = 3#4 co_text["catch-word"], img_boundary = update_region_contours(co_text["catch-word"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "page-number" in elements_with_artificial_class: erosion_rate = 0 - dilation_rate = 2#4 + dilation_rate = 3#4 co_text["page-number"], img_boundary = update_region_contours(co_text["page-number"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "header" in elements_with_artificial_class: - erosion_rate = 0#1 - dilation_rate = 3#4 + erosion_rate = 1 + dilation_rate = 4 co_text["header"], img_boundary = update_region_contours(co_text["header"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "heading" in elements_with_artificial_class: - erosion_rate = 0#1 - dilation_rate = 3#4 + erosion_rate = 1 + dilation_rate = 4 co_text["heading"], img_boundary = update_region_contours(co_text["heading"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "signature-mark" in elements_with_artificial_class: erosion_rate = 1 dilation_rate = 4 co_text["signature-mark"], img_boundary = update_region_contours(co_text["signature-mark"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "marginalia" in elements_with_artificial_class: - erosion_rate = 0#2 - dilation_rate = 3#4 + erosion_rate = 2 + dilation_rate = 4 co_text["marginalia"], img_boundary = update_region_contours(co_text["marginalia"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) if "footnote" in elements_with_artificial_class: erosion_rate = 0#2 From 2d83b8faad8e6e0983529cda221eb17ebb0048f4 Mon Sep 17 00:00:00 2001 From: Clemens Neudecker <952378+cneud@users.noreply.github.com> Date: Thu, 8 Aug 2024 16:35:06 +0200 Subject: [PATCH 083/374] add documentation from wiki as markdown file to the codebase --- train/train.md | 576 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 576 insertions(+) create mode 100644 train/train.md diff --git a/train/train.md b/train/train.md new file mode 100644 index 0000000..553522b --- /dev/null +++ b/train/train.md @@ -0,0 +1,576 @@ +# Documentation for Training Models + +This repository assists users in preparing training datasets, training models, and performing inference with trained models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training dataset. +All these use cases are now utilized in the Eynollah workflow. +As mentioned, the following three tasks can be accomplished using this repository: + +* Generate training dataset +* Train a model +* Inference with the trained model + +## Generate training dataset +The script generate_gt_for_training.py is used for generating training datasets. As the results of the following command demonstrate, the dataset generator provides three different commands: + +`python generate_gt_for_training.py --help` + + +These three commands are: + +* image-enhancement +* machine-based-reading-order +* pagexml2label + + +### image-enhancement + +Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of high-resolution images. The training dataset can then be generated using the following command: + +`python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs "degrading scales json file"` + +The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose resolution at different scales. The degraded images are used as input images, and the original high-resolution images serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: + +```yaml +{ + "scales": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] +} +``` + +### machine-based-reading-order + +For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's input is a three-channel image: the first and last channels contain information about each of the two text regions, while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct reading order. + +For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area to the image area, with a default value of zero. To run the dataset generator, use the following command: + + +`python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` + +### pagexml2label + +pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. +To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. + +In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired element is automatically encoded as 1 in the PNG label. + +To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. For example, in the case of 'textline' detection, the JSON file would resemble this: + +```yaml +{ +"use_case": "textline" +} +``` + +In the case of layout segmentation a possible custom config json file can be like this: + +```yaml +{ +"use_case": "layout", +"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, +"imageregion":4, +"separatorregion":5, +"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} +} +``` + +A possible custom config json file for layout segmentation where the "printspace" is wished to be a class: + +```yaml +{ +"use_case": "layout", +"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, +"imageregion":4, +"separatorregion":5, +"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} +"printspace_as_class_in_layout" : 8 +} +``` +For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', and 'tableregion'. + +Text regions and graphic regions also have their own specific types. The known types for us for text regions are 'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and 'signature'. +Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. + +In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator region" are also present in the label. However, other regions like "noise region" and "table region" will not be included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. + +`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" "` + +We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, the example JSON config file should look like this: + +```yaml +{ + "use_case": "layout", + "textregions": { + "paragraph": 1, + "drop-capital": 1, + "header": 2, + "heading": 2, + "marginalia": 3 + }, + "imageregion": 4, + "separatorregion": 5, + "graphicregions": { + "rest_as_decoration": 6 + }, + "artificial_class_on_boundary": ["paragraph", "header", "heading", "marginalia"], + "artificial_class_label": 7 +} +``` + +This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the elements labeled as "paragraph," "header," "heading," and "marginalia." + +For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the "artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use case: + +```yaml +{ + "use_case": "textline", + "artificial_class_label": 2 +} +``` + +If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that in this scenario, since cropping will be applied to the label files, the directory of the original images must be provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels required for training are obtained. The command should resemble the following: + +`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will be written" ` + +## Train a model +### classification + +For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, all we require is a training directory with subdirectories, each containing images of its respective classes. We need separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both directories. Additionally, the class names should be specified in the config JSON file, as shown in the following example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the "classification_classes_name" key in the config file should appear as follows: + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "classification", + "n_classes" : 2, + "n_epochs" : 10, + "input_height" : 448, + "input_width" : 448, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "f1_threshold_classification": 0.8, + "pretraining" : true, + "classification_classes_name" : {"0":"apple", "1":"orange"}, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +The "dir_train" should be like this: + +``` +. +└── train # train directory + ├── apple # directory of images for apple class + └── orange # directory of images for orange class +``` + +And the "dir_eval" the same structure as train directory: + +``` +. +└── eval # evaluation directory + ├── apple # directory of images for apple class + └── orange # directory of images for orange class + +``` + +The classification model can be trained using the following command line: + +`python train.py with config_classification.json` + + +As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". + +### reading order +An example config json file for machine based reading order should be like this: + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "reading_order", + "n_classes" : 1, + "n_epochs" : 5, + "input_height" : 672, + "input_width" : 448, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "pretraining" : true, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +The "dir_train" should be like this: + +``` +. +└── train # train directory + ├── images # directory of images + └── labels # directory of labels +``` + +And the "dir_eval" the same structure as train directory: + +``` +. +└── eval # evaluation directory + ├── images # directory of images + └── labels # directory of labels +``` + +The classification model can be trained like the classification case command line. + +### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement + +#### Parameter configuration for segmentation or enhancement usecases + +The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. + +* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. +* task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". +* patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be set to ``false``. +* n_batch: Number of batches at each iteration. +* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. +* n_epochs: Number of epochs. +* input_height: This indicates the height of model's input. +* input_width: This indicates the width of model's input. +* weight_decay: Weight decay of l2 regularization of model layers. +* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved in a folder named "pretrained_model" in the same directory of "train.py" script. +* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. +* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. +* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. +* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" parameter. +* degrading: If ``true``, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. +* brightening: If ``true``, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. +* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. +* rotation: If ``true``, 90 degree rotation will be applied on image. +* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. +* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. +* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. +* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. +* flip_index: Type of flips. +* blur_k: Type of blurrings. +* scales: Scales of scaling. +* brightness: The amount of brightenings. +* thetha: Rotation angles. +* degrade_scales: The amount of degradings. +* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. +* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. +* index_start: Starting index for saved models in the case that "continue_training" is ``true``. +* dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. +* transformer_num_patches_xy: Number of patches for vision transformer in x and y direction respectively. +* transformer_patchsize_x: Patch size of vision transformer patches in x direction. +* transformer_patchsize_y: Patch size of vision transformer patches in y direction. +* transformer_projection_dim: Transformer projection dimension. Default value is 64. +* transformer_mlp_head_units: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. +* transformer_layers: transformer layers. Default value is 8. +* transformer_num_heads: Transformer number of heads. Default value is 4. +* transformer_cnn_first: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. + +In the case of segmentation and enhancement the train and evaluation directory should be as following. + +The "dir_train" should be like this: + +``` +. +└── train # train directory + ├── images # directory of images + └── labels # directory of labels +``` + +And the "dir_eval" the same structure as train directory: + +``` +. +└── eval # evaluation directory + ├── images # directory of images + └── labels # directory of labels +``` + +After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following command, similar to the process for classification and reading order: + +`python train.py with config_classification.json` + +#### Binarization + +An example config json file for binarization can be like this: + +```yaml +{ + "backbone_type" : "transformer", + "task": "binarization", + "n_classes" : 2, + "n_epochs" : 4, + "input_height" : 224, + "input_width" : 672, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "transformer_num_patches_xy": [7, 7], + "transformer_patchsize_x": 3, + "transformer_patchsize_y": 1, + "transformer_projection_dim": 192, + "transformer_mlp_head_units": [128, 64], + "transformer_layers": 8, + "transformer_num_heads": 4, + "transformer_cnn_first": true, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +#### Textline + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "segmentation", + "n_classes" : 2, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +#### Enhancement + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "enhancement", + "n_classes" : 3, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel image. + +#### Page extraction + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "segmentation", + "n_classes" : 2, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : false, + "pretraining" : true, + "augmentation" : false, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, hence the patches parameter should be set to false. + +#### layout segmentation + +An example config json file for layout segmentation with 5 classes (including background) can be like this: + +```yaml +{ + "backbone_type" : "transformer", + "task": "segmentation", + "n_classes" : 5, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "transformer_num_patches_xy": [7, 14], + "transformer_patchsize_x": 1, + "transformer_patchsize_y": 1, + "transformer_projection_dim": 64, + "transformer_mlp_head_units": [128, 64], + "transformer_layers": 8, + "transformer_num_heads": 4, + "transformer_cnn_first": true, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` +## Inference with the trained model +### classification + +For conducting inference with a trained model, you simply need to execute the following command line, specifying the directory of the model and the image on which to perform inference: + + +`python inference.py -m "model dir" -i "image" ` + +This will straightforwardly return the class of the image. + +### machine based reading order + + +To infer the reading order using an reading order model, we need a page XML file containing layout information but without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The new XML file with the added reading order will be written to the output directory with the same name. We need to run: + +`python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` + + +### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement + +For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: + + +`python inference.py -m "model dir" -i "image" -p -s "output image" ` + + +Note that in the case of page extraction the -p flag is not needed. + +For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be calculated for the output. To do this, you need to provide the GT label using the argument -gt. + + + From 3b90347a94521f6ed935ab1a94b39fe9504442ce Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 9 Aug 2024 12:46:18 +0200 Subject: [PATCH 084/374] save only layout output. different from overlayed layout on image --- train/inference.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/train/inference.py b/train/inference.py index 49bebf8..6054b01 100644 --- a/train/inference.py +++ b/train/inference.py @@ -32,6 +32,7 @@ class sbb_predict: self.image=image self.patches=patches self.save=save + self.save_layout=save_layout self.model_dir=model self.ground_truth=ground_truth self.task=task @@ -181,6 +182,7 @@ class sbb_predict: prediction = prediction * -1 prediction = prediction + 1 added_image = prediction * 255 + layout_only = None else: unique_classes = np.unique(prediction[:,:,0]) rgb_colors = {'0' : [255, 255, 255], @@ -200,26 +202,26 @@ class sbb_predict: '14' : [255, 125, 125], '15' : [255, 0, 255]} - output = np.zeros(prediction.shape) + layout_only = np.zeros(prediction.shape) for unq_class in unique_classes: rgb_class_unique = rgb_colors[str(int(unq_class))] - output[:,:,0][prediction[:,:,0]==unq_class] = rgb_class_unique[0] - output[:,:,1][prediction[:,:,0]==unq_class] = rgb_class_unique[1] - output[:,:,2][prediction[:,:,0]==unq_class] = rgb_class_unique[2] + layout_only[:,:,0][prediction[:,:,0]==unq_class] = rgb_class_unique[0] + layout_only[:,:,1][prediction[:,:,0]==unq_class] = rgb_class_unique[1] + layout_only[:,:,2][prediction[:,:,0]==unq_class] = rgb_class_unique[2] - img = self.resize_image(img, output.shape[0], output.shape[1]) + img = self.resize_image(img, layout_only.shape[0], layout_only.shape[1]) - output = output.astype(np.int32) + layout_only = layout_only.astype(np.int32) img = img.astype(np.int32) - added_image = cv2.addWeighted(img,0.5,output,0.1,0) + added_image = cv2.addWeighted(img,0.5,layout_only,0.1,0) - return added_image, output + return added_image, layout_only def predict(self): self.start_new_session_and_model() @@ -559,13 +561,12 @@ class sbb_predict: pass elif self.task == 'enhancement': if self.save: - print(self.save) cv2.imwrite(self.save,res) else: - img_seg_overlayed, only_prediction = self.visualize_model_output(res, self.img_org, self.task) + img_seg_overlayed, only_layout = self.visualize_model_output(res, self.img_org, self.task) if self.save: cv2.imwrite(self.save,img_seg_overlayed) - cv2.imwrite('./layout.png', only_prediction) + cv2.imwrite(self.save_layout, only_layout) if self.ground_truth: gt_img=cv2.imread(self.ground_truth) @@ -595,6 +596,11 @@ class sbb_predict: "-s", help="save prediction as a png file in current folder.", ) +@click.option( + "--save_layout", + "-sl", + help="save layout prediction only as a png file in current folder.", +) @click.option( "--model", "-m", @@ -618,7 +624,7 @@ class sbb_predict: "-min", help="min area size of regions considered for reading order detection. The default value is zero and means that all text regions are considered for reading order.", ) -def main(image, model, patches, save, ground_truth, xml_file, out, min_area): +def main(image, model, patches, save, save_layout, ground_truth, xml_file, out, min_area): with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) task = config_params_model['task'] @@ -626,7 +632,7 @@ def main(image, model, patches, save, ground_truth, xml_file, out, min_area): if not save: print("Error: You used one of segmentation or binarization task but not set -s, you need a filename to save visualized output with -s") sys.exit(1) - x=sbb_predict(image, model, task, config_params_model, patches, save, ground_truth, xml_file, out, min_area) + x=sbb_predict(image, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area) x.run() if __name__=="__main__": From bf5837bf6e4c44add1d401a9912fd1bd599df780 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 9 Aug 2024 13:20:09 +0200 Subject: [PATCH 085/374] update --- train/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/inference.py b/train/inference.py index 6054b01..8d0a572 100644 --- a/train/inference.py +++ b/train/inference.py @@ -28,7 +28,7 @@ Tool to load model and predict for given image. """ class sbb_predict: - def __init__(self,image, model, task, config_params_model, patches, save, ground_truth, xml_file, out, min_area): + def __init__(self,image, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area): self.image=image self.patches=patches self.save=save From 5e1821a7419bc20ff760eafccfb940b0c4938eb5 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 21 Aug 2024 00:48:30 +0200 Subject: [PATCH 086/374] augmentation function for red textlines, rgb background and scaling for no patch case --- train/utils.py | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/train/utils.py b/train/utils.py index 891ee15..2278849 100644 --- a/train/utils.py +++ b/train/utils.py @@ -12,6 +12,76 @@ from tensorflow.keras.utils import to_categorical from PIL import Image, ImageEnhance +def return_shuffled_channels(img, channels_order): + """ + channels order in ordinary case is like this [0, 1, 2]. In the case of shuffling the order should be provided. + """ + img_sh = np.copy(img) + + img_sh[:,:,0]= img[:,:,channels_order[0]] + img_sh[:,:,1]= img[:,:,channels_order[1]] + img_sh[:,:,2]= img[:,:,channels_order[2]] + return img_sh + +def return_binary_image_with_red_textlines(img_bin): + img_red = np.copy(img_bin) + + img_red[:,:,0][img_bin[:,:,0] == 0] = 255 + return img_red + +def return_binary_image_with_given_rgb_background(img_bin, img_rgb_background): + img_rgb_background = resize_image(img_rgb_background ,img_bin.shape[0], img_bin.shape[1]) + + img_final = np.copy(img_bin) + + img_final[:,:,0][img_bin[:,:,0] != 0] = img_rgb_background[:,:,0][img_bin[:,:,0] != 0] + img_final[:,:,1][img_bin[:,:,1] != 0] = img_rgb_background[:,:,1][img_bin[:,:,1] != 0] + img_final[:,:,2][img_bin[:,:,2] != 0] = img_rgb_background[:,:,2][img_bin[:,:,2] != 0] + + return img_final + +def return_binary_image_with_given_rgb_background_red_textlines(img_bin, img_rgb_background, img_color): + img_rgb_background = resize_image(img_rgb_background ,img_bin.shape[0], img_bin.shape[1]) + + img_final = np.copy(img_color) + + img_final[:,:,0][img_bin[:,:,0] != 0] = img_rgb_background[:,:,0][img_bin[:,:,0] != 0] + img_final[:,:,1][img_bin[:,:,1] != 0] = img_rgb_background[:,:,1][img_bin[:,:,1] != 0] + img_final[:,:,2][img_bin[:,:,2] != 0] = img_rgb_background[:,:,2][img_bin[:,:,2] != 0] + + return img_final + +def scale_image_for_no_patch(img, label, scale): + h_n = int(img.shape[0]*scale) + w_n = int(img.shape[1]*scale) + + channel0_avg = int( np.mean(img[:,:,0]) ) + channel1_avg = int( np.mean(img[:,:,1]) ) + channel2_avg = int( np.mean(img[:,:,2]) ) + + h_diff = img.shape[0] - h_n + w_diff = img.shape[1] - w_n + + h_start = int(h_diff / 2.) + w_start = int(w_diff / 2.) + + img_res = resize_image(img, h_n, w_n) + label_res = resize_image(label, h_n, w_n) + + img_scaled_padded = np.copy(img) + + label_scaled_padded = np.zeros(label.shape) + + img_scaled_padded[:,:,0] = channel0_avg + img_scaled_padded[:,:,1] = channel1_avg + img_scaled_padded[:,:,2] = channel2_avg + + img_scaled_padded[h_start:h_start+h_n, w_start:w_start+w_n,:] = img_res[:,:,:] + label_scaled_padded[h_start:h_start+h_n, w_start:w_start+w_n,:] = label_res[:,:,:] + + return img_scaled_padded, label_scaled_padded + + def return_number_of_total_training_data(path_classes): sub_classes = os.listdir(path_classes) n_tot = 0 From 445c45cb87935b73099d1753957c4c6c6eac32f2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 21 Aug 2024 16:17:59 +0200 Subject: [PATCH 087/374] updating augmentations --- train/train.py | 8 +++++--- train/utils.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/train/train.py b/train/train.py index 71f31f3..fa08a98 100644 --- a/train/train.py +++ b/train/train.py @@ -53,6 +53,7 @@ def config_params(): degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json. brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. + rgb_background = False dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". dir_output = None # Directory where the output model will be saved. @@ -95,7 +96,7 @@ def run(_config, n_classes, n_epochs, input_height, index_start, dir_of_start_model, is_loss_soft_dice, n_batch, patches, augmentation, flip_aug, blur_aug, padding_white, padding_black, scaling, degrading, - brightening, binarization, blur_k, scales, degrade_scales, + brightening, binarization, rgb_background, blur_k, scales, degrade_scales, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, continue_training, transformer_projection_dim, @@ -108,6 +109,7 @@ def run(_config, n_classes, n_epochs, input_height, if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') dir_eval_flowing = os.path.join(dir_output, 'eval') + dir_flow_train_imgs = os.path.join(dir_train_flowing, 'images') dir_flow_train_labels = os.path.join(dir_train_flowing, 'labels') @@ -161,7 +163,7 @@ def run(_config, n_classes, n_epochs, input_height, # writing patches into a sub-folder in order to be flowed from directory. provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, - blur_aug, padding_white, padding_black, flip_aug, binarization, + blur_aug, padding_white, padding_black, flip_aug, binarization, rgb_background, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=augmentation, @@ -169,7 +171,7 @@ def run(_config, n_classes, n_epochs, input_height, provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, - blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, + blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, rgb_background, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches) diff --git a/train/utils.py b/train/utils.py index 2278849..cf7a65c 100644 --- a/train/utils.py +++ b/train/utils.py @@ -695,6 +695,47 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) indexer += 1 + + if rotation_not_90: + for thetha_i in thetha: + img_max_rotated, label_max_rotated = rotation_not_90_func(cv2.imread(dir_img + '/'+im), + cv2.imread(dir_of_label_file), thetha_i) + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_max_rotated, input_height, input_width)) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_max_rotated, input_height, input_width)) + indexer += 1 + + if channels_shuffling: + for shuffle_index in shuffle_indexes: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + (resize_image(return_shuffled_channels(cv2.imread(dir_img + '/' + im), shuffle_index), input_height, input_width))) + + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + indexer += 1 + + if scaling: + for sc_ind in scales: + img_scaled, label_scaled = scale_image_for_no_patch(cv2.imread(dir_img + '/'+im), + cv2.imread(dir_of_label_file), sc_ind) + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_scaled, input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_scaled, input_height, input_width)) + indexer += 1 + + if rgb_color_background: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + img_with_overlayed_background = return_binary_image_with_given_rgb_background(img_bin_corr, img_rgb_background) + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_with_overlayed_background, input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + + if patches: From aeb2ee4e3ef404b0fef2414462b9e51e9036bc18 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 21 Aug 2024 19:33:23 +0200 Subject: [PATCH 088/374] scaling, channels shuffling, rgb background and red content added to no patch augmentation --- train/config_params.json | 30 +++++++++++++++++++----------- train/train.py | 32 ++++++++++++++++++++++---------- train/utils.py | 32 +++++++++++++++++++++++++++----- 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index a89cbb5..e5f652d 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -1,19 +1,22 @@ { "backbone_type" : "transformer", - "task": "binarization", + "task": "segmentation", "n_classes" : 2, - "n_epochs" : 2, - "input_height" : 224, - "input_width" : 224, + "n_epochs" : 0, + "input_height" : 448, + "input_width" : 448, "weight_decay" : 1e-6, "n_batch" : 1, "learning_rate": 1e-4, - "patches" : true, + "patches" : false, "pretraining" : true, - "augmentation" : false, + "augmentation" : true, "flip_aug" : false, "blur_aug" : false, "scaling" : true, + "adding_rgb_background": true, + "add_red_textlines": true, + "channels_shuffling": true, "degrading": false, "brightening": false, "binarization" : false, @@ -31,18 +34,23 @@ "transformer_num_heads": 1, "transformer_cnn_first": false, "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "scales" : [0.6, 0.7, 0.8, 0.9], "brightness" : [1.3, 1.5, 1.7, 2], "degrade_scales" : [0.2, 0.4], "flip_index" : [0, 1, -1], - "thetha" : [10, -10], + "shuffle_indexes" : [ [0,2,1], [1,2,0], [1,0,2] , [2,1,0]], + "thetha" : [5, -5], + "number_of_backgrounds_per_image": 2, "continue_training": false, "index_start" : 0, "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" + "dir_train": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/train_new", + "dir_eval": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/eval_new", + "dir_output": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/output_new", + "dir_rgb_backgrounds": "/home/vahid/Documents/1_2_test_eynollah/set_rgb_background", + "dir_img_bin": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/train_new/images_bin" + } diff --git a/train/train.py b/train/train.py index fa08a98..5dfad07 100644 --- a/train/train.py +++ b/train/train.py @@ -53,7 +53,9 @@ def config_params(): degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json. brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. - rgb_background = False + adding_rgb_background = False + add_red_textlines = False + channels_shuffling = False dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". dir_eval = None # Directory of validation dataset with subdirectories having the names "images" and "labels". dir_output = None # Directory where the output model will be saved. @@ -65,6 +67,7 @@ def config_params(): scaling_brightness = False # If true, a combination of scaling and brightening will be applied to the image. scaling_flip = False # If true, a combination of scaling and flipping will be applied to the image. thetha = None # Rotate image by these angles for augmentation. + shuffle_indexes = None blur_k = None # Blur image for augmentation. scales = None # Scale patches for augmentation. degrade_scales = None # Degrade image for augmentation. @@ -88,6 +91,10 @@ def config_params(): f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. classification_classes_name = None # Dictionary of classification classes names. backbone_type = None # As backbone we have 2 types of backbones. A vision transformer alongside a CNN and we call it "transformer" and only CNN called "nontransformer" + + dir_img_bin = None + number_of_backgrounds_per_image = 1 + dir_rgb_backgrounds = None @ex.automain @@ -95,15 +102,20 @@ def run(_config, n_classes, n_epochs, input_height, input_width, weight_decay, weighted_loss, index_start, dir_of_start_model, is_loss_soft_dice, n_batch, patches, augmentation, flip_aug, - blur_aug, padding_white, padding_black, scaling, degrading, - brightening, binarization, rgb_background, blur_k, scales, degrade_scales, + blur_aug, padding_white, padding_black, scaling, degrading,channels_shuffling, + brightening, binarization, adding_rgb_background, add_red_textlines, blur_k, scales, degrade_scales,shuffle_indexes, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, continue_training, transformer_projection_dim, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_cnn_first, transformer_patchsize_x, transformer_patchsize_y, transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, - pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name): + pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds): + + if dir_rgb_backgrounds: + list_all_possible_background_images = os.listdir(dir_rgb_backgrounds) + else: + list_all_possible_background_images = None if task == "segmentation" or task == "enhancement" or task == "binarization": if data_is_provided: @@ -163,18 +175,18 @@ def run(_config, n_classes, n_epochs, input_height, # writing patches into a sub-folder in order to be flowed from directory. provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, - blur_aug, padding_white, padding_black, flip_aug, binarization, rgb_background, + blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background,add_red_textlines, channels_shuffling, scaling, degrading, brightening, scales, degrade_scales, brightness, - flip_index, scaling_bluring, scaling_brightness, scaling_binarization, + flip_index,shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=augmentation, - patches=patches) + patches=patches, dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds) provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, - blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, rgb_background, + blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background, add_red_textlines, channels_shuffling, scaling, degrading, brightening, scales, degrade_scales, brightness, - flip_index, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches) + flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, + rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches,dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds) if weighted_loss: weights = np.zeros(n_classes) diff --git a/train/utils.py b/train/utils.py index cf7a65c..20fda29 100644 --- a/train/utils.py +++ b/train/utils.py @@ -51,6 +51,16 @@ def return_binary_image_with_given_rgb_background_red_textlines(img_bin, img_rgb return img_final +def return_image_with_red_elements(img, img_bin): + img_final = np.copy(img) + + img_final[:,:,0][img_bin[:,:,0]==0] = 0 + img_final[:,:,1][img_bin[:,:,0]==0] = 0 + img_final[:,:,2][img_bin[:,:,0]==0] = 255 + return img_final + + + def scale_image_for_no_patch(img, label, scale): h_n = int(img.shape[0]*scale) w_n = int(img.shape[1]*scale) @@ -631,10 +641,10 @@ def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, i def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, blur_aug, - padding_white, padding_black, flip_aug, binarization, scaling, degrading, - brightening, scales, degrade_scales, brightness, flip_index, + padding_white, padding_black, flip_aug, binarization, adding_rgb_background, add_red_textlines, channels_shuffling, scaling, degrading, + brightening, scales, degrade_scales, brightness, flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False): + rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False, dir_img_bin=None,number_of_backgrounds_per_image=None,list_all_possible_background_images=None, dir_rgb_backgrounds=None): indexer = 0 for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): @@ -724,17 +734,29 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_scaled, input_height, input_width)) indexer += 1 - if rgb_color_background: + if adding_rgb_background: img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') for i_n in range(number_of_backgrounds_per_image): background_image_chosen_name = random.choice(list_all_possible_background_images) img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) - img_with_overlayed_background = return_binary_image_with_given_rgb_background(img_bin_corr, img_rgb_background) + img_with_overlayed_background = return_binary_image_with_given_rgb_background(img_bin_corr, img_rgb_background_chosen) cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_with_overlayed_background, input_height, input_width)) cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + indexer += 1 + + if add_red_textlines: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + img_red_context = return_image_with_red_elements(cv2.imread(dir_img + '/'+im), img_bin_corr) + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_red_context, input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + + indexer += 1 + From 61cdd2acb85e65ee023807ad885f1724e476596d Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 22 Aug 2024 21:58:09 +0200 Subject: [PATCH 089/374] using prepared binarized images in the case of augmentation --- train/utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/train/utils.py b/train/utils.py index 20fda29..84af85e 100644 --- a/train/utils.py +++ b/train/utils.py @@ -690,8 +690,15 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow pass if binarization: - cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', - resize_image(otsu_copy(cv2.imread(dir_img + '/' + im)), input_height, input_width)) + + if dir_img_bin: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(img_bin_corr, input_height, input_width)) + else: + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', + resize_image(otsu_copy(cv2.imread(dir_img + '/' + im)), input_height, input_width)) cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) From 5bbd0980b2a1ff3b5aa536353c21241539f6cf7b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 28 Aug 2024 00:04:19 +0200 Subject: [PATCH 090/374] early dilation for textline artificial class --- train/gt_gen_utils.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 13010bf..dd4091f 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -88,12 +88,15 @@ def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) return contours_imgs -def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len): +def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=None): co_text_eroded = [] for con in co_text: img_boundary_in = np.zeros( (y_len,x_len) ) img_boundary_in = cv2.fillPoly(img_boundary_in, pts=[con], color=(1, 1, 1)) + if dilation_early: + img_boundary_in = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_early) + #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica if erosion_rate > 0: img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=erosion_rate) @@ -258,22 +261,25 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "artificial_class_label" in keys: img_boundary = np.zeros((y_len, x_len)) - erosion_rate = 1 + erosion_rate = 0#1 dilation_rate = 3 - co_use_case, img_boundary = update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + dilation_early = 2 + co_use_case, img_boundary = update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=dilation_early ) img = np.zeros((y_len, x_len, 3)) if output_type == '2d': img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) if "artificial_class_label" in keys: - img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label + img_mask = np.copy(img_poly) + img_poly[:,:][(img_boundary[:,:]==1) & (img_mask[:,:,0]!=1)] = artificial_class_label elif output_type == '3d': img_poly = cv2.fillPoly(img, pts=co_use_case, color=textline_rgb_color) if "artificial_class_label" in keys: - img_poly[:,:,0][img_boundary[:,:]==1] = artificial_class_rgb_color[0] - img_poly[:,:,1][img_boundary[:,:]==1] = artificial_class_rgb_color[1] - img_poly[:,:,2][img_boundary[:,:]==1] = artificial_class_rgb_color[2] + img_mask = np.copy(img_poly) + img_poly[:,:,0][(img_boundary[:,:]==1) & (img_mask[:,:,0]!=255)] = artificial_class_rgb_color[0] + img_poly[:,:,1][(img_boundary[:,:]==1) & (img_mask[:,:,0]!=255)] = artificial_class_rgb_color[1] + img_poly[:,:,2][(img_boundary[:,:]==1) & (img_mask[:,:,0]!=255)] = artificial_class_rgb_color[2] if printspace and config_params['use_case']!='printspace': From a57a31673d78741c5679aac66e06991e46fcec73 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 28 Aug 2024 02:09:27 +0200 Subject: [PATCH 091/374] adding foreground rgb to augmentation --- train/config_params.json | 10 ++++++---- train/train.py | 19 +++++++++++++------ train/utils.py | 40 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/train/config_params.json b/train/config_params.json index e5f652d..1db8026 100644 --- a/train/config_params.json +++ b/train/config_params.json @@ -13,13 +13,14 @@ "augmentation" : true, "flip_aug" : false, "blur_aug" : false, - "scaling" : true, + "scaling" : false, "adding_rgb_background": true, - "add_red_textlines": true, - "channels_shuffling": true, + "adding_rgb_foreground": true, + "add_red_textlines": false, + "channels_shuffling": false, "degrading": false, "brightening": false, - "binarization" : false, + "binarization" : true, "scaling_bluring" : false, "scaling_binarization" : false, "scaling_flip" : false, @@ -51,6 +52,7 @@ "dir_eval": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/eval_new", "dir_output": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/output_new", "dir_rgb_backgrounds": "/home/vahid/Documents/1_2_test_eynollah/set_rgb_background", + "dir_rgb_foregrounds": "/home/vahid/Documents/1_2_test_eynollah/out_set_rgb_foreground", "dir_img_bin": "/home/vahid/Documents/test/sbb_pixelwise_segmentation/test_label/pageextractor_test/train_new/images_bin" } diff --git a/train/train.py b/train/train.py index 5dfad07..848ff6a 100644 --- a/train/train.py +++ b/train/train.py @@ -54,6 +54,7 @@ def config_params(): brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. adding_rgb_background = False + adding_rgb_foreground = False add_red_textlines = False channels_shuffling = False dir_train = None # Directory of training dataset with subdirectories having the names "images" and "labels". @@ -95,6 +96,7 @@ def config_params(): dir_img_bin = None number_of_backgrounds_per_image = 1 dir_rgb_backgrounds = None + dir_rgb_foregrounds = None @ex.automain @@ -103,20 +105,25 @@ def run(_config, n_classes, n_epochs, input_height, index_start, dir_of_start_model, is_loss_soft_dice, n_batch, patches, augmentation, flip_aug, blur_aug, padding_white, padding_black, scaling, degrading,channels_shuffling, - brightening, binarization, adding_rgb_background, add_red_textlines, blur_k, scales, degrade_scales,shuffle_indexes, + brightening, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, blur_k, scales, degrade_scales,shuffle_indexes, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, continue_training, transformer_projection_dim, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_cnn_first, transformer_patchsize_x, transformer_patchsize_y, transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, - pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds): + pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds, dir_rgb_foregrounds): if dir_rgb_backgrounds: list_all_possible_background_images = os.listdir(dir_rgb_backgrounds) else: list_all_possible_background_images = None + if dir_rgb_foregrounds: + list_all_possible_foreground_rgbs = os.listdir(dir_rgb_foregrounds) + else: + list_all_possible_foreground_rgbs = None + if task == "segmentation" or task == "enhancement" or task == "binarization": if data_is_provided: dir_train_flowing = os.path.join(dir_output, 'train') @@ -175,18 +182,18 @@ def run(_config, n_classes, n_epochs, input_height, # writing patches into a sub-folder in order to be flowed from directory. provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, - blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background,add_red_textlines, channels_shuffling, + blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background,adding_rgb_foreground, add_red_textlines, channels_shuffling, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index,shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=augmentation, - patches=patches, dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds) + patches=patches, dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds, dir_rgb_foregrounds=dir_rgb_foregrounds,list_all_possible_foreground_rgbs=list_all_possible_foreground_rgbs) provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, - blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background, add_red_textlines, channels_shuffling, + blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, - rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches,dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds) + rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches,dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds,dir_rgb_foregrounds=dir_rgb_foregrounds,list_all_possible_foreground_rgbs=list_all_possible_foreground_rgbs ) if weighted_loss: weights = np.zeros(n_classes) diff --git a/train/utils.py b/train/utils.py index 84af85e..d38e798 100644 --- a/train/utils.py +++ b/train/utils.py @@ -40,6 +40,25 @@ def return_binary_image_with_given_rgb_background(img_bin, img_rgb_background): return img_final +def return_binary_image_with_given_rgb_background_and_given_foreground_rgb(img_bin, img_rgb_background, rgb_foreground): + img_rgb_background = resize_image(img_rgb_background ,img_bin.shape[0], img_bin.shape[1]) + + img_final = np.copy(img_bin) + img_foreground = np.zeros(img_bin.shape) + + + img_foreground[:,:,0][img_bin[:,:,0] == 0] = rgb_foreground[0] + img_foreground[:,:,1][img_bin[:,:,0] == 0] = rgb_foreground[1] + img_foreground[:,:,2][img_bin[:,:,0] == 0] = rgb_foreground[2] + + + img_final[:,:,0][img_bin[:,:,0] != 0] = img_rgb_background[:,:,0][img_bin[:,:,0] != 0] + img_final[:,:,1][img_bin[:,:,1] != 0] = img_rgb_background[:,:,1][img_bin[:,:,1] != 0] + img_final[:,:,2][img_bin[:,:,2] != 0] = img_rgb_background[:,:,2][img_bin[:,:,2] != 0] + + img_final = img_final + img_foreground + return img_final + def return_binary_image_with_given_rgb_background_red_textlines(img_bin, img_rgb_background, img_color): img_rgb_background = resize_image(img_rgb_background ,img_bin.shape[0], img_bin.shape[1]) @@ -641,10 +660,10 @@ def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, i def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, blur_aug, - padding_white, padding_black, flip_aug, binarization, adding_rgb_background, add_red_textlines, channels_shuffling, scaling, degrading, + padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, scaling, degrading, brightening, scales, degrade_scales, brightness, flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, - rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False, dir_img_bin=None,number_of_backgrounds_per_image=None,list_all_possible_background_images=None, dir_rgb_backgrounds=None): + rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False, dir_img_bin=None,number_of_backgrounds_per_image=None,list_all_possible_background_images=None, dir_rgb_backgrounds=None, dir_rgb_foregrounds=None, list_all_possible_foreground_rgbs=None): indexer = 0 for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): @@ -754,6 +773,23 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow indexer += 1 + if adding_rgb_foreground: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + foreground_rgb_chosen_name = random.choice(list_all_possible_foreground_rgbs) + + img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + foreground_rgb_chosen = np.load(dir_rgb_foregrounds + '/' + foreground_rgb_chosen_name) + + img_with_overlayed_background = return_binary_image_with_given_rgb_background_and_given_foreground_rgb(img_bin_corr, img_rgb_background_chosen, foreground_rgb_chosen) + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_with_overlayed_background, input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', + resize_image(cv2.imread(dir_of_label_file), input_height, input_width)) + + indexer += 1 + if add_red_textlines: img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') img_red_context = return_image_with_red_elements(cv2.imread(dir_img + '/'+im), img_bin_corr) From e3da4944704d9d4af22a008addc1df8183a6ef44 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 28 Aug 2024 17:34:06 +0200 Subject: [PATCH 092/374] fixing artificial class bug --- train/gt_gen_utils.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index dd4091f..5784e14 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -8,6 +8,7 @@ from tqdm import tqdm import cv2 from shapely import geometry from pathlib import Path +import matplotlib.pyplot as plt KERNEL = np.ones((5, 5), np.uint8) @@ -83,9 +84,13 @@ def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): ret, thresh = cv2.threshold(imgray, 0, 255, 0) contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + #print(len(contours_imgs), hierarchy) contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) + + #print(len(contours_imgs), "iki") + #contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) return contours_imgs def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=None): @@ -103,12 +108,15 @@ def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y pixel = 1 min_size = 0 + + img_boundary_in = img_boundary_in.astype("uint8") + con_eroded = return_contours_of_interested_region(img_boundary_in,pixel, min_size ) - try: - co_text_eroded.append(con_eroded[0]) - except: - co_text_eroded.append(con) + #try: + co_text_eroded.append(con_eroded[0]) + #except: + #co_text_eroded.append(con) img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_rate) @@ -262,8 +270,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if "artificial_class_label" in keys: img_boundary = np.zeros((y_len, x_len)) erosion_rate = 0#1 - dilation_rate = 3 - dilation_early = 2 + dilation_rate = 2 + dilation_early = 1 co_use_case, img_boundary = update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=dilation_early ) From 3f354e1c342a36d52883c61bacebcddf43a31c54 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 30 Aug 2024 15:30:18 +0200 Subject: [PATCH 093/374] new augmentations for patchwise training --- train/utils.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/train/utils.py b/train/utils.py index d38e798..3d42b64 100644 --- a/train/utils.py +++ b/train/utils.py @@ -823,6 +823,53 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow img_max_rotated, label_max_rotated, input_height, input_width, indexer=indexer) + + if channels_shuffling: + for shuffle_index in shuffle_indexes: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + return_shuffled_channels(cv2.imread(dir_img + '/' + im), shuffle_index), + cv2.imread(dir_of_label_file), + input_height, input_width, indexer=indexer) + + if adding_rgb_background: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + img_with_overlayed_background = return_binary_image_with_given_rgb_background(img_bin_corr, img_rgb_background_chosen) + + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + img_with_overlayed_background, + cv2.imread(dir_of_label_file), + input_height, input_width, indexer=indexer) + + + if adding_rgb_foreground: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + for i_n in range(number_of_backgrounds_per_image): + background_image_chosen_name = random.choice(list_all_possible_background_images) + foreground_rgb_chosen_name = random.choice(list_all_possible_foreground_rgbs) + + img_rgb_background_chosen = cv2.imread(dir_rgb_backgrounds + '/' + background_image_chosen_name) + foreground_rgb_chosen = np.load(dir_rgb_foregrounds + '/' + foreground_rgb_chosen_name) + + img_with_overlayed_background = return_binary_image_with_given_rgb_background_and_given_foreground_rgb(img_bin_corr, img_rgb_background_chosen, foreground_rgb_chosen) + + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + img_with_overlayed_background, + cv2.imread(dir_of_label_file), + input_height, input_width, indexer=indexer) + + + if add_red_textlines: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + img_red_context = return_image_with_red_elements(cv2.imread(dir_img + '/'+im), img_bin_corr) + + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + img_red_context, + cv2.imread(dir_of_label_file), + input_height, input_width, indexer=indexer) + if flip_aug: for f_i in flip_index: indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, @@ -871,10 +918,19 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow input_height, input_width, indexer=indexer) if binarization: - indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, - otsu_copy(cv2.imread(dir_img + '/' + im)), - cv2.imread(dir_of_label_file), - input_height, input_width, indexer=indexer) + if dir_img_bin: + img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') + + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + img_bin_corr, + cv2.imread(dir_of_label_file), + input_height, input_width, indexer=indexer) + + else: + indexer = get_patches(dir_flow_train_imgs, dir_flow_train_labels, + otsu_copy(cv2.imread(dir_img + '/' + im)), + cv2.imread(dir_of_label_file), + input_height, input_width, indexer=indexer) if scaling_brightness: for sc_ind in scales: From a524f8b1a7e5e68219cdcb12e239bc6ae8a1391c Mon Sep 17 00:00:00 2001 From: johnlockejrr Date: Sat, 19 Oct 2024 13:21:29 -0700 Subject: [PATCH 094/374] Update inference.py to check if save_layout was passed as argument otherwise can give an cv2 error --- train/inference.py | 1 + 1 file changed, 1 insertion(+) diff --git a/train/inference.py b/train/inference.py index 8d0a572..89d32de 100644 --- a/train/inference.py +++ b/train/inference.py @@ -566,6 +566,7 @@ class sbb_predict: img_seg_overlayed, only_layout = self.visualize_model_output(res, self.img_org, self.task) if self.save: cv2.imwrite(self.save,img_seg_overlayed) + if self.save_layout: cv2.imwrite(self.save_layout, only_layout) if self.ground_truth: From f09eed1197d3f4d6cb4672fec48f73f50a1eee6b Mon Sep 17 00:00:00 2001 From: johnlockejrr Date: Sat, 19 Oct 2024 13:25:50 -0700 Subject: [PATCH 095/374] Changed deprecated `lr` to `learning_rate` and `model.fit_generator` to `model.fit` --- train/train.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/train/train.py b/train/train.py index 848ff6a..4cc3cbb 100644 --- a/train/train.py +++ b/train/train.py @@ -277,16 +277,16 @@ def run(_config, n_classes, n_epochs, input_height, if (task == "segmentation" or task == "binarization"): if not is_loss_soft_dice and not weighted_loss: model.compile(loss='categorical_crossentropy', - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) if is_loss_soft_dice: model.compile(loss=soft_dice_loss, - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) if weighted_loss: model.compile(loss=weighted_categorical_crossentropy(weights), - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) elif task == "enhancement": model.compile(loss='mean_squared_error', - optimizer=Adam(lr=learning_rate), metrics=['accuracy']) + optimizer=Adam(learning_rate=learning_rate), metrics=['accuracy']) # generating train and evaluation data @@ -299,7 +299,7 @@ def run(_config, n_classes, n_epochs, input_height, ##score_best=[] ##score_best.append(0) for i in tqdm(range(index_start, n_epochs + index_start)): - model.fit_generator( + model.fit( train_gen, steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, validation_data=val_gen, @@ -384,7 +384,7 @@ def run(_config, n_classes, n_epochs, input_height, #f1score_tot = [0] indexer_start = 0 - opt = SGD(lr=0.01, momentum=0.9) + opt = SGD(learning_rate=0.01, momentum=0.9) opt_adam = tf.keras.optimizers.Adam(learning_rate=0.0001) model.compile(loss="binary_crossentropy", optimizer = opt_adam,metrics=['accuracy']) From fd14e656aa38b17ca25224268d2e66634506b107 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 25 Oct 2024 14:01:39 +0200 Subject: [PATCH 096/374] early_erosion is added --- train/gt_gen_utils.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 5784e14..cabc7df 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -93,7 +93,7 @@ def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): #contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) return contours_imgs -def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=None): +def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=None, erosion_early=None): co_text_eroded = [] for con in co_text: img_boundary_in = np.zeros( (y_len,x_len) ) @@ -101,6 +101,9 @@ def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y if dilation_early: img_boundary_in = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_early) + + if erosion_early: + img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=erosion_early) #img_boundary_in = cv2.erode(img_boundary_in[:,:], KERNEL, iterations=7)#asiatica if erosion_rate > 0: @@ -137,6 +140,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ ls_org_imgs_stem = [item.split('.')[0] for item in ls_org_imgs] for index in tqdm(range(len(gt_list))): #try: + print(gt_list[index]) tree1 = ET.parse(dir_in+'/'+gt_list[index], parser = ET.XMLParser(encoding = 'iso-8859-5')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] @@ -271,8 +275,9 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_boundary = np.zeros((y_len, x_len)) erosion_rate = 0#1 dilation_rate = 2 - dilation_early = 1 - co_use_case, img_boundary = update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=dilation_early ) + dilation_early = 0 + erosion_early = 2 + co_use_case, img_boundary = update_region_contours(co_use_case, img_boundary, erosion_rate, dilation_rate, y_len, x_len, dilation_early=dilation_early, erosion_early=erosion_early) img = np.zeros((y_len, x_len, 3)) @@ -280,7 +285,8 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly = cv2.fillPoly(img, pts=co_use_case, color=(1, 1, 1)) if "artificial_class_label" in keys: img_mask = np.copy(img_poly) - img_poly[:,:][(img_boundary[:,:]==1) & (img_mask[:,:,0]!=1)] = artificial_class_label + ##img_poly[:,:][(img_boundary[:,:]==1) & (img_mask[:,:,0]!=1)] = artificial_class_label + img_poly[:,:][img_boundary[:,:]==1] = artificial_class_label elif output_type == '3d': img_poly = cv2.fillPoly(img, pts=co_use_case, color=textline_rgb_color) if "artificial_class_label" in keys: From 7b4d14b19f536614545b209bf3834b6b84a67d1d Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 29 Oct 2024 17:06:22 +0100 Subject: [PATCH 097/374] addinh shifting augmentation --- train/train.py | 7 ++++--- train/utils.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/train/train.py b/train/train.py index 848ff6a..7e3e390 100644 --- a/train/train.py +++ b/train/train.py @@ -50,6 +50,7 @@ def config_params(): padding_white = False # If true, white padding will be applied to the image. padding_black = False # If true, black padding will be applied to the image. scaling = False # If true, scaling will be applied to the image. The amount of scaling is defined with "scales" in config_params.json. + shifting = False degrading = False # If true, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" in config_params.json. brightening = False # If true, brightening will be applied to the image. The amount of brightening is defined with "brightness" in config_params.json. binarization = False # If true, Otsu thresholding will be applied to augment the input with binarized images. @@ -104,7 +105,7 @@ def run(_config, n_classes, n_epochs, input_height, input_width, weight_decay, weighted_loss, index_start, dir_of_start_model, is_loss_soft_dice, n_batch, patches, augmentation, flip_aug, - blur_aug, padding_white, padding_black, scaling, degrading,channels_shuffling, + blur_aug, padding_white, padding_black, scaling, shifting, degrading,channels_shuffling, brightening, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, blur_k, scales, degrade_scales,shuffle_indexes, brightness, dir_train, data_is_provided, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, @@ -183,7 +184,7 @@ def run(_config, n_classes, n_epochs, input_height, provide_patches(imgs_list, segs_list, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background,adding_rgb_foreground, add_red_textlines, channels_shuffling, - scaling, degrading, brightening, scales, degrade_scales, brightness, + scaling, shifting, degrading, brightening, scales, degrade_scales, brightness, flip_index,shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=augmentation, patches=patches, dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds, dir_rgb_foregrounds=dir_rgb_foregrounds,list_all_possible_foreground_rgbs=list_all_possible_foreground_rgbs) @@ -191,7 +192,7 @@ def run(_config, n_classes, n_epochs, input_height, provide_patches(imgs_list_test, segs_list_test, dir_img_val, dir_seg_val, dir_flow_eval_imgs, dir_flow_eval_labels, input_height, input_width, blur_k, blur_aug, padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, - scaling, degrading, brightening, scales, degrade_scales, brightness, + scaling, shifting, degrading, brightening, scales, degrade_scales, brightness, flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=patches,dir_img_bin=dir_img_bin,number_of_backgrounds_per_image=number_of_backgrounds_per_image,list_all_possible_background_images=list_all_possible_background_images, dir_rgb_backgrounds=dir_rgb_backgrounds,dir_rgb_foregrounds=dir_rgb_foregrounds,list_all_possible_foreground_rgbs=list_all_possible_foreground_rgbs ) diff --git a/train/utils.py b/train/utils.py index 3d42b64..d7ddb99 100644 --- a/train/utils.py +++ b/train/utils.py @@ -78,7 +78,50 @@ def return_image_with_red_elements(img, img_bin): img_final[:,:,2][img_bin[:,:,0]==0] = 255 return img_final +def shift_image_and_label(img, label, type_shift): + h_n = int(img.shape[0]*1.06) + w_n = int(img.shape[1]*1.06) + + channel0_avg = int( np.mean(img[:,:,0]) ) + channel1_avg = int( np.mean(img[:,:,1]) ) + channel2_avg = int( np.mean(img[:,:,2]) ) + h_diff = abs( img.shape[0] - h_n ) + w_diff = abs( img.shape[1] - w_n ) + + h_start = int(h_diff / 2.) + w_start = int(w_diff / 2.) + + img_scaled_padded = np.zeros((h_n, w_n, 3)) + label_scaled_padded = np.zeros((h_n, w_n, 3)) + + img_scaled_padded[:,:,0] = channel0_avg + img_scaled_padded[:,:,1] = channel1_avg + img_scaled_padded[:,:,2] = channel2_avg + + img_scaled_padded[h_start:h_start+img.shape[0], w_start:w_start+img.shape[1],:] = img[:,:,:] + label_scaled_padded[h_start:h_start+img.shape[0], w_start:w_start+img.shape[1],:] = label[:,:,:] + + + if type_shift=="xpos": + img_dis = img_scaled_padded[h_start:h_start+img.shape[0],2*w_start:2*w_start+img.shape[1],:] + label_dis = label_scaled_padded[h_start:h_start+img.shape[0],2*w_start:2*w_start+img.shape[1],:] + elif type_shift=="xmin": + img_dis = img_scaled_padded[h_start:h_start+img.shape[0],:img.shape[1],:] + label_dis = label_scaled_padded[h_start:h_start+img.shape[0],:img.shape[1],:] + elif type_shift=="ypos": + img_dis = img_scaled_padded[2*h_start:2*h_start+img.shape[0],w_start:w_start+img.shape[1],:] + label_dis = label_scaled_padded[2*h_start:2*h_start+img.shape[0],w_start:w_start+img.shape[1],:] + elif type_shift=="ymin": + img_dis = img_scaled_padded[:img.shape[0],w_start:w_start+img.shape[1],:] + label_dis = label_scaled_padded[:img.shape[0],w_start:w_start+img.shape[1],:] + elif type_shift=="xypos": + img_dis = img_scaled_padded[2*h_start:2*h_start+img.shape[0],2*w_start:2*w_start+img.shape[1],:] + label_dis = label_scaled_padded[2*h_start:2*h_start+img.shape[0],2*w_start:2*w_start+img.shape[1],:] + elif type_shift=="xymin": + img_dis = img_scaled_padded[:img.shape[0],:img.shape[1],:] + label_dis = label_scaled_padded[:img.shape[0],:img.shape[1],:] + return img_dis, label_dis def scale_image_for_no_patch(img, label, scale): h_n = int(img.shape[0]*scale) @@ -660,7 +703,7 @@ def get_patches_num_scale_new(dir_img_f, dir_seg_f, img, label, height, width, i def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow_train_imgs, dir_flow_train_labels, input_height, input_width, blur_k, blur_aug, - padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, scaling, degrading, + padding_white, padding_black, flip_aug, binarization, adding_rgb_background, adding_rgb_foreground, add_red_textlines, channels_shuffling, scaling, shifting, degrading, brightening, scales, degrade_scales, brightness, flip_index, shuffle_indexes, scaling_bluring, scaling_brightness, scaling_binarization, rotation, rotation_not_90, thetha, scaling_flip, task, augmentation=False, patches=False, dir_img_bin=None,number_of_backgrounds_per_image=None,list_all_possible_background_images=None, dir_rgb_backgrounds=None, dir_rgb_foregrounds=None, list_all_possible_foreground_rgbs=None): @@ -759,6 +802,16 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_scaled, input_height, input_width)) cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_scaled, input_height, input_width)) indexer += 1 + if shifting: + shift_types = ['xpos', 'xmin', 'ypos', 'ymin', 'xypos', 'xymin'] + for st_ind in shift_types: + img_shifted, label_shifted = shift_image_and_label(cv2.imread(dir_img + '/'+im), + cv2.imread(dir_of_label_file), st_ind) + + cv2.imwrite(dir_flow_train_imgs + '/img_' + str(indexer) + '.png', resize_image(img_shifted, input_height, input_width)) + cv2.imwrite(dir_flow_train_labels + '/img_' + str(indexer) + '.png', resize_image(label_shifted, input_height, input_width)) + indexer += 1 + if adding_rgb_background: img_bin_corr = cv2.imread(dir_img_bin + '/' + img_name+'.png') From 238ea3bd8ef59da890646c9b1581145b8d937d85 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 14 Nov 2024 16:26:19 +0100 Subject: [PATCH 098/374] update resizing in inference --- train/inference.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/train/inference.py b/train/inference.py index 8d0a572..2b12ff7 100644 --- a/train/inference.py +++ b/train/inference.py @@ -442,10 +442,11 @@ class sbb_predict: self.img_org = np.copy(img) if img.shape[0] < self.img_height: - img = cv2.resize(img, (img.shape[1], self.img_width), interpolation=cv2.INTER_NEAREST) + img = self.resize_image(img, self.img_height, img.shape[1]) if img.shape[1] < self.img_width: - img = cv2.resize(img, (self.img_height, img.shape[0]), interpolation=cv2.INTER_NEAREST) + img = self.resize_image(img, img.shape[0], self.img_width) + margin = int(0.1 * self.img_width) width_mid = self.img_width - 2 * margin height_mid = self.img_height - 2 * margin From e9b860b27513a255ec94892aec8b6a61e23d0b87 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 18 Nov 2024 16:34:53 +0100 Subject: [PATCH 099/374] artificial_class_label for table region --- train/gt_gen_utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index cabc7df..95b8414 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -116,10 +116,10 @@ def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y con_eroded = return_contours_of_interested_region(img_boundary_in,pixel, min_size ) - #try: - co_text_eroded.append(con_eroded[0]) - #except: - #co_text_eroded.append(con) + try: + co_text_eroded.append(con_eroded[0]) + except: + co_text_eroded.append(con) img_boundary_in_dilated = cv2.dilate(img_boundary_in[:,:], KERNEL, iterations=dilation_rate) @@ -636,6 +636,10 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ erosion_rate = 0#2 dilation_rate = 2#4 co_text["footnote-continued"], img_boundary = update_region_contours(co_text["footnote-continued"], img_boundary, erosion_rate, dilation_rate, y_len, x_len ) + if "tableregion" in elements_with_artificial_class: + erosion_rate = 0#2 + dilation_rate = 3#4 + co_table, img_boundary = update_region_contours(co_table, img_boundary, erosion_rate, dilation_rate, y_len, x_len ) From 90a1b186f78a9ad5934c4d46d93e1c2bf20d6789 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 14 Mar 2025 17:20:33 +0100 Subject: [PATCH 100/374] this enables to visualize reading order of textregions provided in page-xml files --- train/generate_gt_for_training.py | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index cfcc151..9e0f45e 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -214,6 +214,73 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'.png' ), input_multi_visual_modal) indexer = indexer+1 + + +@main.command() +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out", + "-do", + help="directory where plots will be written", + type=click.Path(exists=True, file_okay=False), +) + + +def visualize_reading_order(dir_xml, dir_out): + xml_files_ind = os.listdir(dir_xml) + + + indexer_start= 0#55166 + #min_area = 0.0001 + + for ind_xml in tqdm(xml_files_ind): + indexer = 0 + #print(ind_xml) + #print('########################') + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = ind_xml.split('.')[0] + _, _, _, file_name, id_paragraph, id_header,co_text_paragraph,co_text_header,tot_region_ref,x_len, y_len,index_tot_regions,img_poly = read_xml(xml_file) + + id_all_text = id_paragraph + id_header + co_text_all = co_text_paragraph + co_text_header + + + cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours(co_text_all) + + texts_corr_order_index = [int(index_tot_regions[tot_region_ref.index(i)]) for i in id_all_text ] + #texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] + + + #cx_ordered = np.array(cx_main)[np.array(texts_corr_order_index)] + #cx_ordered = cx_ordered.astype(np.int32) + + cx_ordered = [int(val) for (_, val) in sorted(zip(texts_corr_order_index, cx_main), key=lambda x: \ + x[0], reverse=False)] + #cx_ordered = cx_ordered.astype(np.int32) + + cy_ordered = [int(val) for (_, val) in sorted(zip(texts_corr_order_index, cy_main), key=lambda x: \ + x[0], reverse=False)] + #cy_ordered = cy_ordered.astype(np.int32) + + + color = (0, 0, 255) + thickness = 20 + + img = np.zeros( (y_len,x_len,3) ) + img = cv2.fillPoly(img, pts =co_text_all, color=(255,0,0)) + for i in range(len(cx_ordered)-1): + start_point = (int(cx_ordered[i]), int(cy_ordered[i])) + end_point = (int(cx_ordered[i+1]), int(cy_ordered[i+1])) + img = cv2.arrowedLine(img, start_point, end_point, + color, thickness, tipLength = 0.03) + + cv2.imwrite(os.path.join(dir_out, f_name+'.png'), img) From 363c343b373d99170d795ff20520ba9e586b4ab1 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 17 Mar 2025 20:09:48 +0100 Subject: [PATCH 101/374] visualising reaidng order- Overlaying on image is provided --- train/generate_gt_for_training.py | 36 ++++++++++++++------- train/gt_gen_utils.py | 53 +++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 9e0f45e..9869bfa 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -231,8 +231,12 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i type=click.Path(exists=True, file_okay=False), ) +@click.option( + "--dir_imgs", + "-dimg", + help="directory where the overlayed plots will be written", ) -def visualize_reading_order(dir_xml, dir_out): +def visualize_reading_order(dir_xml, dir_out, dir_imgs): xml_files_ind = os.listdir(dir_xml) @@ -271,16 +275,26 @@ def visualize_reading_order(dir_xml, dir_out): color = (0, 0, 255) thickness = 20 - - img = np.zeros( (y_len,x_len,3) ) - img = cv2.fillPoly(img, pts =co_text_all, color=(255,0,0)) - for i in range(len(cx_ordered)-1): - start_point = (int(cx_ordered[i]), int(cy_ordered[i])) - end_point = (int(cx_ordered[i+1]), int(cy_ordered[i+1])) - img = cv2.arrowedLine(img, start_point, end_point, - color, thickness, tipLength = 0.03) - - cv2.imwrite(os.path.join(dir_out, f_name+'.png'), img) + if dir_imgs: + layout = np.zeros( (y_len,x_len,3) ) + layout = cv2.fillPoly(layout, pts =co_text_all, color=(1,1,1)) + + img_file_name_with_format = find_format_of_given_filename_in_dir(dir_imgs, f_name) + img = cv2.imread(os.path.join(dir_imgs, img_file_name_with_format)) + + overlayed = overlay_layout_on_image(layout, img, cx_ordered, cy_ordered, color, thickness) + cv2.imwrite(os.path.join(dir_out, f_name+'.png'), overlayed) + + else: + img = np.zeros( (y_len,x_len,3) ) + img = cv2.fillPoly(img, pts =co_text_all, color=(255,0,0)) + for i in range(len(cx_ordered)-1): + start_point = (int(cx_ordered[i]), int(cy_ordered[i])) + end_point = (int(cx_ordered[i+1]), int(cy_ordered[i+1])) + img = cv2.arrowedLine(img, start_point, end_point, + color, thickness, tipLength = 0.03) + + cv2.imwrite(os.path.join(dir_out, f_name+'.png'), img) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 95b8414..753abf2 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -1290,3 +1290,56 @@ def update_list_and_return_first_with_length_bigger_than_one(index_element_to_be else: early_list_bigger_than_one = -20 return list_inp, early_list_bigger_than_one + +def overlay_layout_on_image(prediction, img, cx_ordered, cy_ordered, color, thickness): + + unique_classes = np.unique(prediction[:,:,0]) + rgb_colors = {'0' : [255, 255, 255], + '1' : [255, 0, 0], + '2' : [0, 0, 255], + '3' : [255, 0, 125], + '4' : [125, 125, 125], + '5' : [125, 125, 0], + '6' : [0, 125, 255], + '7' : [0, 125, 0], + '8' : [125, 125, 125], + '9' : [0, 125, 255], + '10' : [125, 0, 125], + '11' : [0, 255, 0], + '12' : [255, 125, 0], + '13' : [0, 255, 255], + '14' : [255, 125, 125], + '15' : [255, 0, 255]} + + layout_only = np.zeros(prediction.shape) + + for unq_class in unique_classes: + rgb_class_unique = rgb_colors[str(int(unq_class))] + layout_only[:,:,0][prediction[:,:,0]==unq_class] = rgb_class_unique[0] + layout_only[:,:,1][prediction[:,:,0]==unq_class] = rgb_class_unique[1] + layout_only[:,:,2][prediction[:,:,0]==unq_class] = rgb_class_unique[2] + + + + #img = self.resize_image(img, layout_only.shape[0], layout_only.shape[1]) + + layout_only = layout_only.astype(np.int32) + + for i in range(len(cx_ordered)-1): + start_point = (int(cx_ordered[i]), int(cy_ordered[i])) + end_point = (int(cx_ordered[i+1]), int(cy_ordered[i+1])) + layout_only = cv2.arrowedLine(layout_only, start_point, end_point, + color, thickness, tipLength = 0.03) + + img = img.astype(np.int32) + + + + added_image = cv2.addWeighted(img,0.5,layout_only,0.1,0) + + return added_image + +def find_format_of_given_filename_in_dir(dir_imgs, f_name): + ls_imgs = os.listdir(dir_imgs) + file_interested = [ind for ind in ls_imgs if ind.startswith(f_name+'.')] + return file_interested[0] From a22df11ebb564631611f4609048b31e67eb0541f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 14 Apr 2025 00:42:08 +0200 Subject: [PATCH 102/374] Restoring the contour in the original image caused an error due to an empty tuple. This issue has been resolved, and as expected, the confidence score for this contour is set to zero --- src/eynollah/utils/contour.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index a81ccb4..0e84153 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -230,7 +230,6 @@ def get_textregion_contours_in_org_image_light_old(cnts, img, slope_first): def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first, confidence_matrix): img_copy = np.zeros(img.shape) img_copy = cv2.fillPoly(img_copy, pts=[contour_par], color=(1, 1, 1)) - confidence_matrix_mapped_with_contour = confidence_matrix * img_copy[:,:,0] confidence_contour = np.sum(confidence_matrix_mapped_with_contour) / float(np.sum(img_copy[:,:,0])) @@ -239,9 +238,13 @@ def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first ret, thresh = cv2.threshold(imgray, 0, 255, 0) cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) - cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) - # print(np.shape(cont_int[0])) + if len(cont_int)==0: + cont_int = [] + cont_int.append(contour_par) + confidence_contour = 0 + else: + cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) + cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) return cont_int[0], index_r_con, confidence_contour def get_textregion_contours_in_org_image_light(cnts, img, slope_first, confidence_matrix, map=map): From 41318f0404722c3980db6f9174871c2e222258d7 Mon Sep 17 00:00:00 2001 From: kba Date: Tue, 15 Apr 2025 11:14:26 +0200 Subject: [PATCH 103/374] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f7ce6bb..ad86fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Fixed: + + * restoring the contour in the original image caused an error due to an empty tuple + ## [0.4.0] - 2025-04-07 Fixed: From 30ba23464193b61541e4ba7784974b4d5c4ec33d Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 16 Apr 2025 19:27:17 +0200 Subject: [PATCH 104/374] CI: pypi --- .github/workflows/pypi.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/pypi.yml diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 0000000..bb2344e --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,24 @@ +name: PyPI CD + +on: + release: + types: [published] + workflow_dispatch: + +jobs: + pypi-publish: + name: upload release to PyPI + runs-on: ubuntu-latest + permissions: + # IMPORTANT: this permission is mandatory for Trusted Publishing + id-token: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + - name: Build package + run: make build + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true From 825b2634f96788cc3351f089d24b8a1c2e202194 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 16 Apr 2025 23:36:41 +0200 Subject: [PATCH 105/374] rotation augmentation is provided for machine based reading order --- train/train.py | 7 +++++-- train/utils.py | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/train/train.py b/train/train.py index 7e3e390..130c7f4 100644 --- a/train/train.py +++ b/train/train.py @@ -380,7 +380,10 @@ def run(_config, n_classes, n_epochs, input_height, dir_flow_train_labels = os.path.join(dir_train, 'labels') classes = os.listdir(dir_flow_train_labels) - num_rows =len(classes) + if augmentation: + num_rows = len(classes)*(len(thetha) + 1) + else: + num_rows = len(classes) #ls_test = os.listdir(dir_flow_train_labels) #f1score_tot = [0] @@ -390,7 +393,7 @@ def run(_config, n_classes, n_epochs, input_height, model.compile(loss="binary_crossentropy", optimizer = opt_adam,metrics=['accuracy']) for i in range(n_epochs): - history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes), steps_per_epoch=num_rows / n_batch, verbose=1) + history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes, thetha, augmentation), steps_per_epoch=num_rows / n_batch, verbose=1) model.save( os.path.join(dir_output,'model_'+str(i+indexer_start) )) with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: diff --git a/train/utils.py b/train/utils.py index d7ddb99..50c21af 100644 --- a/train/utils.py +++ b/train/utils.py @@ -363,6 +363,11 @@ def rotation_not_90_func(img, label, thetha): return rotate_max_area(img, rotated, rotated_label, thetha) +def rotation_not_90_func_single_image(img, thetha): + rotated = imutils.rotate(img, thetha) + return rotate_max_area(img, rotated, thetha) + + def color_images(seg, n_classes): ann_u = range(n_classes) if len(np.shape(seg)) == 3: @@ -410,7 +415,7 @@ def IoU(Yi, y_predi): #print("Mean IoU: {:4.3f}".format(mIoU)) return mIoU -def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batchsize, height, width, n_classes): +def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batchsize, height, width, n_classes, thetha, augmentation=False): all_labels_files = os.listdir(classes_file_dir) ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) @@ -433,6 +438,22 @@ def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batch ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) batchcount = 0 + + if augmentation: + for thetha_i in thetha: + img_rot = rotation_not_90_func_single_image(img, thetha_i) + + ret_x[batchcount, :,:,0] = img_rot[:,:,0]/3.0 + ret_x[batchcount, :,:,2] = img_rot[:,:,2]/3.0 + ret_x[batchcount, :,:,1] = img_rot[:,:,1]/5.0 + + ret_y[batchcount, :] = label_class + batchcount+=1 + if batchcount>=batchsize: + yield (ret_x, ret_y) + ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) + ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) + batchcount = 0 def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_classes, task='segmentation'): c = 0 From dd21a3b33a3adb1a8ba2c34e2144e01b2b094366 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 17 Apr 2025 00:05:59 +0200 Subject: [PATCH 106/374] updating:rotation augmentation is provided for machine based reading order --- train/utils.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/train/utils.py b/train/utils.py index 50c21af..485056b 100644 --- a/train/utils.py +++ b/train/utils.py @@ -356,6 +356,18 @@ def rotate_max_area(image, rotated, rotated_label, angle): x2 = x1 + int(wr) return rotated[y1:y2, x1:x2], rotated_label[y1:y2, x1:x2] +def rotate_max_area_single_image(image, rotated, angle): + """ image: cv2 image matrix object + angle: in degree + """ + wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], + math.radians(angle)) + h, w, _ = rotated.shape + y1 = h // 2 - int(hr / 2) + y2 = y1 + int(hr) + x1 = w // 2 - int(wr / 2) + x2 = x1 + int(wr) + return rotated[y1:y2, x1:x2] def rotation_not_90_func(img, label, thetha): rotated = imutils.rotate(img, thetha) @@ -365,7 +377,7 @@ def rotation_not_90_func(img, label, thetha): def rotation_not_90_func_single_image(img, thetha): rotated = imutils.rotate(img, thetha) - return rotate_max_area(img, rotated, thetha) + return rotate_max_area_single_image(img, rotated, thetha) def color_images(seg, n_classes): From 4635dd219d5cfade1c038a371dceb78452a7fbf9 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 17 Apr 2025 00:12:30 +0200 Subject: [PATCH 107/374] updating:rotation augmentation is provided for machine based reading order --- train/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/train/utils.py b/train/utils.py index 485056b..8be6963 100644 --- a/train/utils.py +++ b/train/utils.py @@ -455,6 +455,8 @@ def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batch for thetha_i in thetha: img_rot = rotation_not_90_func_single_image(img, thetha_i) + img_rot = resize_image(img_rot, height, width) + ret_x[batchcount, :,:,0] = img_rot[:,:,0]/3.0 ret_x[batchcount, :,:,2] = img_rot[:,:,2]/3.0 ret_x[batchcount, :,:,1] = img_rot[:,:,1]/5.0 From 192b9111e31eee4758364b1fe9f63f80aa533ec2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 22 Apr 2025 00:23:01 +0200 Subject: [PATCH 108/374] updating eynollah README, how to use it for use cases --- README.md | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16ac661..3cfb587 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,16 @@ For documentation on methods and models, have a look at [`models.md`](https://gi In case you want to train your own model with Eynollah, have a look at [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md). ## Usage -The command-line interface can be called like this: + +Eynollah has four key use cases: layout analysis, binarization, OCR, and machine-based reading order. + +### Layout +The layout module is responsible for detecting layouts, identifying text lines, and determining reading order using both heuristic methods or a machine-based reading order detection model. It's important to note that this functionality should not be confused with the machine-based-reading-order use case. The latter, still under development, focuses specifically on determining the reading order for a given layout in an XML file. In contrast, layout detection takes an image as input, and after detecting the layout, it can also determine the reading order using a machine-based model. + +The command-line interface for layout can be called like this: ```sh -eynollah \ +eynollah layout \ -i | -di \ -o \ -m \ @@ -66,6 +72,7 @@ The following options can be used to further configure the processing: |-------------------|:-------------------------------------------------------------------------------| | `-fl` | full layout analysis including all steps and segmentation classes | | `-light` | lighter and faster but simpler method for main region detection and deskewing | +| `-tll` | this indicates the light textline and should be passed with light version | | `-tab` | apply table detection | | `-ae` | apply enhancement (the resulting image is saved to the output directory) | | `-as` | apply scaling | @@ -83,6 +90,34 @@ The following options can be used to further configure the processing: If no option is set, the tool performs layout detection of main regions (background, text, images, separators and marginals). The best output quality is produced when RGB images are used as input rather than greyscale or binarized images. +### Binarization +Document Image Binarization + +The command-line interface for binarization of single image can be called like this: + +```sh +eynollah binarization \ + -m \ + \ + +``` + +and for flowing from a directory like this: + +```sh +eynollah binarization \ + -m \ + -di \ + -do +``` + +### OCR +Under development + +### Machine-based-reading-order +Under development + + #### Use as OCR-D processor Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) [processor](https://ocr-d.de/en/spec/cli), From 77dae129d50783b225eb3f72e32d38adaa8e0610 Mon Sep 17 00:00:00 2001 From: Konstantin Baierer Date: Tue, 22 Apr 2025 13:22:28 +0200 Subject: [PATCH 109/374] CI: Use most recent actions/setup-python@v5 --- .github/workflows/pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index bb2344e..248f4ef 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 - name: Build package run: make build - name: Publish package distributions to PyPI From 208bde706f6a998af7811372ca80be82d3af95cb Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 30 Apr 2025 13:55:09 +0200 Subject: [PATCH 110/374] resolving issue #158 --- src/eynollah/utils/separate_lines.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 3499c29..6602574 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -214,9 +214,13 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): textline_con_fil=filter_contours_area_of_image(img_patch, textline_con, hierarchy, max_area=1, min_area=0.0008) - y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) - sigma_gaus=int( y_diff_mean * (7./40.0) ) - #print(sigma_gaus,'sigma_gaus') + + if len(np.diff(peaks_new_tot))>0: + y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) + sigma_gaus=int( y_diff_mean * (7./40.0) ) + else: + sigma_gaus=12 + except: sigma_gaus=12 if sigma_gaus<3: @@ -1616,6 +1620,7 @@ def do_work_of_slopes_new( textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, hierarchy, max_area=1, min_area=0.00008) + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) if len(textline_con_fil) > 1 else np.NaN if np.isnan(y_diff_mean): slope_for_all = MAX_SLOPE @@ -1641,13 +1646,6 @@ def do_work_of_slopes_new( all_text_region_raw = textline_mask_tot_ea[y: y + h, x: x + w].copy() mask_only_con_region = mask_only_con_region[y: y + h, x: x + w] - ##plt.imshow(textline_mask_tot_ea) - ##plt.show() - ##plt.imshow(all_text_region_raw) - ##plt.show() - ##plt.imshow(mask_only_con_region) - ##plt.show() - all_text_region_raw[mask_only_con_region == 0] = 0 cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text) From 4cb4414740a89741c6ff25a33932ffc16ce201f8 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 30 Apr 2025 16:01:52 +0200 Subject: [PATCH 111/374] Resolve remaining issue with #158 and resolving #124 --- src/eynollah/utils/separate_lines.py | 263 ++++++++++----------------- 1 file changed, 95 insertions(+), 168 deletions(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 6602574..0322579 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -102,14 +102,15 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): textline_con_fil = filter_contours_area_of_image(img_patch, textline_con, hierarchy, max_area=1, min_area=0.0008) - y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) - sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) - # print(sigma_gaus,'sigma_gaus') + if len(np.diff(peaks_new_tot))>1: + y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) + else: + sigma_gaus = 12 except: sigma_gaus = 12 if sigma_gaus < 3: sigma_gaus = 3 - # print(sigma_gaus,'sigma') y_padded_smoothed = gaussian_filter1d(y_padded, sigma_gaus) y_padded_up_to_down = -y_padded + np.max(y_padded) @@ -137,7 +138,6 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): M = cv2.getRotationMatrix2D(center, -thetha, 1.0) x_d = M[0, 2] y_d = M[1, 2] - thetha = thetha / 180. * np.pi rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) contour_text_interest_copy = contour_text_interest.copy() @@ -162,77 +162,73 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): x = np.array(range(len(y))) peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - if 1>0: - try: - y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) - y_padded_up_to_down_e=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) - y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e - y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) - + + try: + y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) + y_padded_up_to_down_e=-y_padded+np.max(y_padded) + y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) + y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e + y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) + + peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) + peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) + neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) - peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) - neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + arg_neg_must_be_deleted= np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3] + diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) + + arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] + peaks_new=peaks_e[:] + peaks_neg_new=peaks_neg_e[:] - arg_neg_must_be_deleted= np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3] - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] + clusters_to_be_deleted=[] + if len(arg_diff_cluster)>0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) + for i in range(len(arg_diff_cluster)-1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1: + arg_diff_cluster[i+1]+1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) + if len(clusters_to_be_deleted)>0: + peaks_new_extra=[] + for m in range(len(clusters_to_be_deleted)): + min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) + max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) + peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] + peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] + peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] + peaks_new_tot=[] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot=np.sort(peaks_new_tot) + else: + peaks_new_tot=peaks_e[:] - peaks_new=peaks_e[:] - peaks_neg_new=peaks_neg_e[:] + textline_con,hierarchy=return_contours_of_image(img_patch) + textline_con_fil=filter_contours_area_of_image(img_patch, + textline_con, hierarchy, + max_area=1, min_area=0.0008) - clusters_to_be_deleted=[] - if len(arg_diff_cluster)>0: - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1: - arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - else: - peaks_new_tot=peaks_e[:] - - textline_con,hierarchy=return_contours_of_image(img_patch) - textline_con_fil=filter_contours_area_of_image(img_patch, - textline_con, hierarchy, - max_area=1, min_area=0.0008) - - if len(np.diff(peaks_new_tot))>0: - y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) - sigma_gaus=int( y_diff_mean * (7./40.0) ) - else: - sigma_gaus=12 - - except: + if len(np.diff(peaks_new_tot))>0: + y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) + sigma_gaus=int( y_diff_mean * (7./40.0) ) + else: sigma_gaus=12 - if sigma_gaus<3: - sigma_gaus=3 - #print(sigma_gaus,'sigma') + + except: + sigma_gaus=12 + if sigma_gaus<3: + sigma_gaus=3 y_padded_smoothed= gaussian_filter1d(y_padded, sigma_gaus) y_padded_up_to_down=-y_padded+np.max(y_padded) y_padded_up_to_down_padded=np.zeros(len(y_padded_up_to_down)+40) y_padded_up_to_down_padded[20:len(y_padded_up_to_down)+20]=y_padded_up_to_down y_padded_up_to_down_padded= gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) - peaks, _ = find_peaks(y_padded_smoothed, height=0) peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) @@ -243,6 +239,7 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] + except: arg_neg_must_be_deleted=[] arg_diff_cluster=[] @@ -250,7 +247,6 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): peaks_new=peaks[:] peaks_neg_new=peaks_neg[:] clusters_to_be_deleted=[] - if len(arg_diff_cluster)>=2 and len(arg_diff_cluster)>0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) for i in range(len(arg_diff_cluster)-1): @@ -279,21 +275,6 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): peaks_new_tot.append(i1) peaks_new_tot=np.sort(peaks_new_tot) - ##plt.plot(y_padded_up_to_down_padded) - ##plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') - ##plt.show() - - ##plt.plot(y_padded_up_to_down_padded) - ##plt.plot(peaks_neg_new,y_padded_up_to_down_padded[peaks_neg_new],'*') - ##plt.show() - - ##plt.plot(y_padded_smoothed) - ##plt.plot(peaks,y_padded_smoothed[peaks],'*') - ##plt.show() - - ##plt.plot(y_padded_smoothed) - ##plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') - ##plt.show() peaks=peaks_new_tot[:] peaks_neg=peaks_neg_new[:] else: @@ -302,11 +283,13 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): peaks_neg=peaks_neg_new[:] except: pass - - mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks=np.std(y_padded_smoothed[peaks]) + if len(y_padded_smoothed[peaks]) > 1: + mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks=np.std(y_padded_smoothed[peaks]) + else: + mean_value_of_peaks = np.nan + std_value_of_peaks = np.nan peaks_values=y_padded_smoothed[peaks] - peaks_neg = peaks_neg - 20 - 20 peaks = peaks - 20 for jj in range(len(peaks_neg)): @@ -349,7 +332,6 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_down_narrow = peaks[jj] + first_nonzero + int( 1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) - if point_down_narrow >= img_patch.shape[0]: point_down_narrow = img_patch.shape[0] - 2 @@ -605,7 +587,6 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) - return peaks, textline_boxes_rot def separate_lines_vertical(img_patch, contour_text_interest, thetha): @@ -637,7 +618,7 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): peaks_neg_new = peaks_neg[:] clusters_to_be_deleted = [] - if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + if len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) >= 2: clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : @@ -645,7 +626,7 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - if len(arg_neg_must_be_deleted) == 1: + else: clusters_to_be_deleted.append(arg_neg_must_be_deleted) if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] @@ -671,9 +652,14 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): peaks_new_tot = peaks[:] peaks = peaks_new_tot[:] peaks_neg = peaks_neg_new[:] - - mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + + if len(y_padded_smoothed[peaks])>1: + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + else: + mean_value_of_peaks = np.nan + std_value_of_peaks = np.nan + peaks_values = y_padded_smoothed[peaks] peaks_neg = peaks_neg - 20 - 20 @@ -691,7 +677,6 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): textline_boxes_rot = [] if len(peaks_neg) == len(peaks) + 1 and len(peaks) >= 3: - # print('11') for jj in range(len(peaks)): if jj == (len(peaks) - 1): @@ -998,15 +983,16 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): textline_con_fil = filter_contours_area_of_image(img_patch, textline_con, hierarchy, max_area=1, min_area=0.0008) - y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + if len(np.diff(peaks_new_tot)): + y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) + else: + sigma_gaus = 12 - sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) - # print(sigma_gaus,'sigma_gaus') except: sigma_gaus = 12 if sigma_gaus < 3: sigma_gaus = 3 - # print(sigma_gaus,'sigma') y_padded_smoothed = gaussian_filter1d(y_padded, sigma_gaus) y_padded_up_to_down = -y_padded + np.max(y_padded) @@ -1030,7 +1016,7 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] clusters_to_be_deleted = [] - if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + if len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) >= 2: clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : @@ -1038,7 +1024,7 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - if len(arg_neg_must_be_deleted) == 1: + else: clusters_to_be_deleted.append(arg_neg_must_be_deleted) if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] @@ -1081,9 +1067,14 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): peaks_new_tot = peaks[:] peaks = peaks_new_tot[:] peaks_neg = peaks_neg_new[:] - - mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + + if len(y_padded_smoothed[peaks]) > 1: + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + else: + mean_value_of_peaks = np.nan + std_value_of_peaks = np.nan + peaks_values = y_padded_smoothed[peaks] ###peaks_neg = peaks_neg - 20 - 20 @@ -1093,10 +1084,8 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): if len(peaks_neg_true) > 0: peaks_neg_true = np.array(peaks_neg_true) - peaks_neg_true = peaks_neg_true - 20 - 20 - # print(peaks_neg_true) for i in range(len(peaks_neg_true)): img_patch[peaks_neg_true[i] - 6 : peaks_neg_true[i] + 6, :] = 0 else: @@ -1181,13 +1170,11 @@ def separate_lines_new_inside_tiles(img_path, thetha): if diff_peaks[i] <= cut_off: forest.append(peaks_neg[i + 1]) if diff_peaks[i] > cut_off: - # print(forest[np.argmin(z[forest]) ] ) if not np.isnan(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) forest = [] forest.append(peaks_neg[i + 1]) if i == (len(peaks_neg) - 1): - # print(print(forest[np.argmin(z[forest]) ] )) if not np.isnan(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) @@ -1204,17 +1191,14 @@ def separate_lines_new_inside_tiles(img_path, thetha): if diff_peaks_pos[i] <= cut_off: forest.append(peaks[i + 1]) if diff_peaks_pos[i] > cut_off: - # print(forest[np.argmin(z[forest]) ] ) if not np.isnan(forest[np.argmax(z[forest])]): peaks_pos_true.append(forest[np.argmax(z[forest])]) forest = [] forest.append(peaks[i + 1]) if i == (len(peaks) - 1): - # print(print(forest[np.argmin(z[forest]) ] )) if not np.isnan(forest[np.argmax(z[forest])]): peaks_pos_true.append(forest[np.argmax(z[forest])]) - # print(len(peaks_neg_true) ,len(peaks_pos_true) ,'lensss') if len(peaks_neg_true) > 0: peaks_neg_true = np.array(peaks_neg_true) @@ -1240,7 +1224,6 @@ def separate_lines_new_inside_tiles(img_path, thetha): """ peaks_neg_true = peaks_neg_true - 20 - 20 - # print(peaks_neg_true) for i in range(len(peaks_neg_true)): img_path[peaks_neg_true[i] - 6 : peaks_neg_true[i] + 6, :] = 0 @@ -1282,7 +1265,6 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i contours_imgs, hierarchy, max_area=max_area, min_area=min_area) cont_final = [] - ###print(add_boxes_coor_into_textlines,'ikki') for i in range(len(contours_imgs)): img_contour = np.zeros((cnts_images.shape[0], cnts_images.shape[1], 3)) img_contour = cv2.fillPoly(img_contour, pts=[contours_imgs[i]], color=(255, 255, 255)) @@ -1297,12 +1279,10 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i ##0] ##contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] ##if add_boxes_coor_into_textlines: - ##print(np.shape(contours_text_rot[0]),'sjppo') ##contours_text_rot[0][:, 0, 0]=contours_text_rot[0][:, 0, 0] + box_ind[0] ##contours_text_rot[0][:, 0, 1]=contours_text_rot[0][:, 0, 1] + box_ind[1] cont_final.append(contours_text_rot[0]) - ##print(cont_final,'nadizzzz') return None, cont_final def textline_contours_postprocessing(textline_mask, slope, contour_text_interest, box_ind, add_boxes_coor_into_textlines=False): @@ -1313,20 +1293,7 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, kernel) textline_mask = cv2.erode(textline_mask, kernel, iterations=2) # textline_mask = cv2.erode(textline_mask, kernel, iterations=1) - - # print(textline_mask.shape[0]/float(textline_mask.shape[1]),'miz') try: - # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - # plt.imshow(textline_mask) - # plt.show() - - # if abs(slope)>1: - # x_help=30 - # y_help=2 - # else: - # x_help=2 - # y_help=2 - x_help = 30 y_help = 2 @@ -1350,28 +1317,12 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest img_contour = np.zeros((box_ind[3], box_ind[2], 3)) img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=(255, 255, 255)) - # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - # plt.imshow(img_contour) - # plt.show() - img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), img_contour.shape[1] + int(2 * x_help), 3)) img_contour_help[y_help : y_help + img_contour.shape[0], x_help : x_help + img_contour.shape[1], :] = np.copy(img_contour[:, :, :]) img_contour_rot = rotate_image(img_contour_help, slope) - # plt.imshow(img_contour_rot_help) - # plt.show() - - # plt.imshow(dst_help) - # plt.show() - - # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - # plt.imshow(img_contour_rot_help) - # plt.show() - - # plt.imshow(dst_help) - # plt.show() img_contour_rot = img_contour_rot.astype(np.uint8) # dst_help = dst_help.astype(np.uint8) @@ -1382,9 +1333,7 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] ind_big_con = np.argmax(len_con_text_rot) - # print('juzaa') if abs(slope) > 45: - # print(add_boxes_coor_into_textlines,'avval') _, contours_rotated_clean = separate_lines_vertical_cont( textline_mask, contours_text_rot[ind_big_con], box_ind, slope, add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) @@ -1416,7 +1365,6 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl length_x = int(img_path.shape[1] / float(num_patches)) # margin = int(0.04 * length_x) just recently this was changed because it break lines into 2 margin = int(0.04 * length_x) - # print(margin,'margin') # if margin<=4: # margin = int(0.08 * length_x) # margin=0 @@ -1456,11 +1404,9 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl # if abs(slope_region)>70 and abs(slope_xline)<25: # slope_xline=[slope_region][0] slopes_tile_wise.append(slope_xline) - # print(slope_xline,'xlineeee') img_line_rotated = rotate_image(img_xline, slope_xline) img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 - - # print(slopes_tile_wise,'slopes_tile_wise') + img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] img_patch_ineterst_revised = np.zeros(img_patch_ineterst.shape) @@ -1502,8 +1448,6 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl img_patch_separated_returned_true_size = img_patch_separated_returned_true_size[:, margin : length_x - margin] img_patch_ineterst_revised[:, index_x_d + margin : index_x_u - margin] = img_patch_separated_returned_true_size - # plt.imshow(img_patch_ineterst_revised) - # plt.show() return img_patch_ineterst_revised def do_image_rotation(angle, img, sigma_des, logger=None): @@ -1536,20 +1480,13 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, #img_resized[ int( img_int.shape[0]*(.4)):int( img_int.shape[0]*(.4))+img_int.shape[0] , int( img_int.shape[1]*(.8)):int( img_int.shape[1]*(.8))+img_int.shape[1] ]=img_int[:,:] img_resized[ onset_y:onset_y+img_int.shape[0] , onset_x:onset_x+img_int.shape[1] ]=img_int[:,:] - #print(img_resized.shape,'img_resizedshape') - #plt.imshow(img_resized) - #plt.show() if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]: - #plt.imshow(img_resized) - #plt.show() angles = np.array([-45, 0, 45, 90,]) angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles) angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) elif main_page: - #plt.imshow(img_resized) - #plt.show() angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) @@ -1620,7 +1557,6 @@ def do_work_of_slopes_new( textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, hierarchy, max_area=1, min_area=0.00008) - y_diff_mean = find_contours_mean_y_diff(textline_con_fil) if len(textline_con_fil) > 1 else np.NaN if np.isnan(y_diff_mean): slope_for_all = MAX_SLOPE @@ -1637,12 +1573,9 @@ def do_work_of_slopes_new( if slope_for_all == MAX_SLOPE: slope_for_all = slope_deskew slope = slope_for_all - mask_only_con_region = np.zeros(textline_mask_tot_ea.shape) mask_only_con_region = cv2.fillPoly(mask_only_con_region, pts=[contour_par], color=(1, 1, 1)) - # plt.imshow(mask_only_con_region) - # plt.show() all_text_region_raw = textline_mask_tot_ea[y: y + h, x: x + w].copy() mask_only_con_region = mask_only_con_region[y: y + h, x: x + w] @@ -1706,20 +1639,15 @@ def do_work_of_slopes_new_curved( mask_region_in_patch_region = mask_biggest[y : y + h, x : x + w] textline_biggest_region = mask_biggest * textline_mask_tot_ea - # print(slope_for_all,'slope_for_all') textline_rotated_separated = separate_lines_new2(textline_biggest_region[y: y+h, x: x+w], 0, num_col, slope_for_all, logger=logger, plotter=plotter) - # new line added - ##print(np.shape(textline_rotated_separated),np.shape(mask_biggest)) + textline_rotated_separated[mask_region_in_patch_region[:, :] != 1] = 0 - # till here textline_region_in_image[y : y + h, x : x + w] = textline_rotated_separated - # plt.imshow(textline_region_in_image) - # plt.show() pixel_img = 1 cnt_textlines_in_image = return_contours_of_interested_textline(textline_region_in_image, pixel_img) @@ -1742,7 +1670,6 @@ def do_work_of_slopes_new_curved( logger.error(why) else: textlines_cnt_per_region = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text, True) - # print(np.shape(textlines_cnt_per_region),'textlines_cnt_per_region') return textlines_cnt_per_region[::-1], box_text, contour, contour_par, crop_coor, index_r_con, slope From b227736094e33e2ba8cd6446eb9c0b46c006c10f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 30 Apr 2025 16:04:34 +0200 Subject: [PATCH 112/374] Fix OCR text cleaning to correctly handle 'U', 'K', and 'N' starting sentence; update text line splitting size --- src/eynollah/eynollah.py | 62 ++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 19 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 022cf0a..a94e890 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -259,7 +259,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_mb_ro_aug_3"#"/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" @@ -3320,12 +3320,22 @@ class Eynollah: def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p): y_len = text_regions_p.shape[0] x_len = text_regions_p.shape[1] + img_poly = np.zeros((y_len,x_len), dtype='uint8') img_poly[text_regions_p[:,:]==1] = 1 img_poly[text_regions_p[:,:]==2] = 2 img_poly[text_regions_p[:,:]==3] = 4 img_poly[text_regions_p[:,:]==6] = 5 + + + #temp + sep_mask = (img_poly==5)*1 + sep_mask = sep_mask.astype('uint8') + sep_mask = cv2.erode(sep_mask, kernel=KERNEL, iterations=2) + img_poly[img_poly==5] = 0 + img_poly[sep_mask==1] = 5 + # img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') if contours_only_text_parent_h: @@ -3341,9 +3351,13 @@ class Eynollah: if not len(co_text_all): return [], [] - labels_con = np.zeros((y_len, x_len, len(co_text_all)), dtype=bool) + labels_con = np.zeros((int(y_len /6.), int(x_len/6.), len(co_text_all)), dtype=bool) + co_text_all = [(i/6).astype(int) for i in co_text_all] for i in range(len(co_text_all)): img = labels_con[:,:,i].astype(np.uint8) + + #img = cv2.resize(img, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) + cv2.fillPoly(img, pts=[co_text_all[i]], color=(1,)) labels_con[:,:,i] = img @@ -3359,6 +3373,7 @@ class Eynollah: labels_con = resize_image(labels_con.astype(np.uint8), height1, width1).astype(bool) img_header_and_sep = resize_image(img_header_and_sep, height1, width1) img_poly = resize_image(img_poly, height3, width3) + inference_bs = 3 input_1 = np.zeros((inference_bs, height1, width1, 3)) @@ -4575,10 +4590,6 @@ class Eynollah: return pcgts - ## check the ro order - - - #print("text region early 3 in %.1fs", time.time() - t0) if self.light_version: @@ -4886,7 +4897,7 @@ class Eynollah_ocr: self.model_ocr.to(self.device) else: - self.model_ocr_dir = dir_models + "/model_step_75000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -4974,7 +4985,7 @@ class Eynollah_ocr: def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image): width = np.shape(textline_image)[1] height = np.shape(textline_image)[0] - common_window = int(0.06*width) + common_window = int(0.22*width) width1 = int ( width/2. - common_window ) width2 = int ( width/2. + common_window ) @@ -4984,13 +4995,17 @@ class Eynollah_ocr: peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: + if len(peaks_real)>35: - peaks_real = peaks_real[(peaks_realwidth1)] + #peaks_real = peaks_real[(peaks_realwidth1)] + argsort = np.argsort(sum_smoothed[peaks_real])[::-1] + peaks_real_top_six = peaks_real[argsort[:6]] + midpoint = textline_image.shape[1] / 2. + arg_closest = np.argmin(np.abs(peaks_real_top_six - midpoint)) - arg_max = np.argmax(sum_smoothed[peaks_real]) + #arg_max = np.argmax(sum_smoothed[peaks_real]) - peaks_final = peaks_real[arg_max] + peaks_final = peaks_real_top_six[arg_closest]#peaks_real[arg_max] return peaks_final else: @@ -5038,10 +5053,19 @@ class Eynollah_ocr: if width_new == 0: width_new = img.shape[1] + + ##if width_new+32 >= image_width: + ##width_new = width_new - 32 + + ###patch_zero = np.zeros((32, 32, 3))#+255 + ###patch_zero[9:19,8:18,:] = 0 + img = resize_image(img, image_height, width_new) img_fin = np.ones((image_height, image_width, 3))*255 - img_fin[:,:+width_new,:] = img[:,:,:] + ###img_fin[:,:32,:] = patch_zero[:,:,:] + ###img_fin[:,32:32+width_new,:] = img[:,:,:] + img_fin[:,:width_new,:] = img[:,:,:] img_fin = img_fin / 255. return img_fin @@ -5097,7 +5121,7 @@ class Eynollah_ocr: img_crop = img_poly_on_img[y:y+h, x:x+w, :] img_crop[mask_poly==0] = 255 - if h2w_ratio > 0.05: + if h2w_ratio > 0.1: cropped_lines.append(img_crop) cropped_lines_meging_indexing.append(0) else: @@ -5234,7 +5258,7 @@ class Eynollah_ocr: if self.draw_texts_on_image: total_bb_coordinates.append([x,y,w,h]) - h2w_ratio = h/float(w) + w_scaled = w * image_height/float(h) img_poly_on_img = np.copy(img) if self.prediction_with_both_of_rgb_and_bin: @@ -5252,7 +5276,7 @@ class Eynollah_ocr: img_crop_bin[mask_poly==0] = 255 if not self.export_textline_images_and_text: - if h2w_ratio > 0.1: + if w_scaled < 1.5*image_width: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) @@ -5334,11 +5358,11 @@ class Eynollah_ocr: if self.prediction_with_both_of_rgb_and_bin: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) preds = (preds + preds_bin) / 2. - + pred_texts = self.decode_batch_predictions(preds) for ib in range(imgs.shape[0]): - pred_texts_ib = pred_texts[ib].strip("[UNK]") + pred_texts_ib = pred_texts[ib].replace("[UNK]", "") extracted_texts.append(pred_texts_ib) extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] @@ -5378,7 +5402,7 @@ class Eynollah_ocr: text_by_textregion = [] for ind in unique_cropped_lines_region_indexer: extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] - text_by_textregion.append(" ".join(extracted_texts_merged_un)) + text_by_textregion.append("".join(extracted_texts_merged_un)) indexer = 0 indexer_textregion = 0 From e2da7a623987f9c693957512f7902947699389ff Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 30 Apr 2025 16:06:29 +0200 Subject: [PATCH 113/374] Fix model name to return the correct machine-based model name --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index a94e890..d47016b 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -259,7 +259,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_mb_ro_aug_3"#"/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" From f8b4d29a59098f8a82e90b2790015568841bc53f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 2 May 2025 00:13:11 +0200 Subject: [PATCH 114/374] docker: prepackage ocrd-all-module-dir.json --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4785fc1..4ba498b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,8 @@ COPY . . COPY ocrd-tool.json . # prepackage ocrd-tool.json as ocrd-all-tool.json RUN ocrd ocrd-tool ocrd-tool.json dump-tools > $(dirname $(ocrd bashlib filename))/ocrd-all-tool.json +# prepackage ocrd-all-module-dir.json +RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib filename))/ocrd-all-module-dir.json # install everything and reduce image size RUN make install EXTRAS=OCR && rm -rf /build/eynollah # smoke test From e9179e1d3458c9261989cf996863881f03a24ebd Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 2 May 2025 00:13:06 +0200 Subject: [PATCH 115/374] docker: use latest core base stage --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5f2bf34..73d4d34 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,9 @@ PIP ?= pip3 EXTRAS ?= # DOCKER_BASE_IMAGE = artefakt.dev.sbb.berlin:5000/sbb/ocrd_core:v2.68.0 -DOCKER_BASE_IMAGE = docker.io/ocrd/core-cuda-tf2:v3.3.0 -DOCKER_TAG = ocrd/eynollah +DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-tf2:latest +DOCKER_TAG ?= ocrd/eynollah +DOCKER ?= docker #SEG_MODEL := https://qurator-data.de/eynollah/2021-04-25/models_eynollah.tar.gz #SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah_renamed.tar.gz @@ -117,7 +118,7 @@ coverage: # Build docker image docker: - docker build \ + $(DOCKER) build \ --build-arg DOCKER_BASE_IMAGE=$(DOCKER_BASE_IMAGE) \ --build-arg VCS_REF=$$(git rev-parse --short HEAD) \ --build-arg BUILD_DATE=$$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ From 184af46664ac05a780f2cad07cc950bb594d9352 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 2 May 2025 00:30:36 +0200 Subject: [PATCH 116/374] displaying detexted text on an image is provided for trocr case --- src/eynollah/eynollah.py | 55 +++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index d47016b..5793d37 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -259,7 +259,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_mb_ro_aug_2"#"/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" @@ -1221,7 +1221,7 @@ class Eynollah: seg_art[seg_art>0] =1 seg_line = label_p_pred[:,:,:,3] - seg_line[seg_line>0.1] =1 + seg_line[seg_line>0.5] =1#seg_line[seg_line>0.1] =1 seg_line[seg_line<1] =0 seg[seg_art==1]=4 @@ -3329,13 +3329,13 @@ class Eynollah: img_poly[text_regions_p[:,:]==6] = 5 - #temp - sep_mask = (img_poly==5)*1 - sep_mask = sep_mask.astype('uint8') - sep_mask = cv2.erode(sep_mask, kernel=KERNEL, iterations=2) - img_poly[img_poly==5] = 0 - img_poly[sep_mask==1] = 5 - # + ###temp + ##sep_mask = (img_poly==5)*1 + ##sep_mask = sep_mask.astype('uint8') + ##sep_mask = cv2.erode(sep_mask, kernel=KERNEL, iterations=2) + ##img_poly[img_poly==5] = 0 + ##img_poly[sep_mask==1] = 5 + ### img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') if contours_only_text_parent_h: @@ -5081,6 +5081,12 @@ class Eynollah_ocr: dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') img = cv2.imread(dir_img) + + if self.draw_texts_on_image: + out_image_with_text = os.path.join(self.dir_out_image_text, file_name+'.png') + image_text = Image.new("RGB", (img.shape[1], img.shape[0]), "white") + draw = ImageDraw.Draw(image_text) + total_bb_coordinates = [] ##file_name = Path(dir_xmls).stem tree1 = ET.parse(dir_xml, parser = ET.XMLParser(encoding="utf-8")) @@ -5111,6 +5117,9 @@ class Eynollah_ocr: textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) x,y,w,h = cv2.boundingRect(textline_coords) + if self.draw_texts_on_image: + total_bb_coordinates.append([x,y,w,h]) + h2w_ratio = h/float(w) img_poly_on_img = np.copy(img) @@ -5161,6 +5170,34 @@ class Eynollah_ocr: #print(extracted_texts_merged, len(extracted_texts_merged)) unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) + + if self.draw_texts_on_image: + + font_path = "NotoSans-Regular.ttf" # Make sure this file exists! + font = ImageFont.truetype(font_path, 40) + + for indexer_text, bb_ind in enumerate(total_bb_coordinates): + + + x_bb = bb_ind[0] + y_bb = bb_ind[1] + w_bb = bb_ind[2] + h_bb = bb_ind[3] + + font = self.fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) + + ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) + + text_bbox = draw.textbbox((0, 0), extracted_texts_merged[indexer_text], font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + + text_x = x_bb + (w_bb - text_width) // 2 # Center horizontally + text_y = y_bb + (h_bb - text_height) // 2 # Center vertically + + # Draw the text + draw.text((text_x, text_y), extracted_texts_merged[indexer_text], fill="black", font=font) + image_text.save(out_image_with_text) #print(len(unique_cropped_lines_region_indexer), 'unique_cropped_lines_region_indexer') text_by_textregion = [] From 5d8c864c0881256d16f8484d01f8e1f34fdad254 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 2 May 2025 01:02:32 +0200 Subject: [PATCH 117/374] adding space between splitted textline predicted text in the case of trocr --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 5793d37..d148c67 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5164,7 +5164,7 @@ class Eynollah_ocr: extracted_texts = extracted_texts + generated_text_merged - extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] #print(extracted_texts_merged, len(extracted_texts_merged)) From a1a004b19da5dfc828fa077f25b10c72722f71c8 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 2 May 2025 12:53:33 +0200 Subject: [PATCH 118/374] inference batch size for ocr is passed as an argument --- src/eynollah/cli.py | 8 +++++- src/eynollah/eynollah.py | 53 ++++++++++++++++++++++++++-------------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index c189aca..56d5d7e 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -374,6 +374,11 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ is_flag=True, help="If this parameter is set to True, the prediction will be performed using both RGB and binary images. However, this does not necessarily improve results; it may be beneficial for certain document images.", ) +@click.option( + "--batch_size", + "-bs", + help="number of inference batch size. Default b_s for trocr and cnn_rnn models are 2 and 8 respectively", +) @click.option( "--log_level", "-l", @@ -381,7 +386,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, log_level): +def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) @@ -397,6 +402,7 @@ def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, ex do_not_mask_with_textline_contour=do_not_mask_with_textline_contour, draw_texts_on_image=draw_texts_on_image, prediction_with_both_of_rgb_and_bin=prediction_with_both_of_rgb_and_bin, + batch_size=batch_size, ) eynollah_ocr.run() diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index d148c67..62026bf 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4872,6 +4872,7 @@ class Eynollah_ocr: dir_out=None, dir_out_image_text=None, tr_ocr=False, + batch_size=None, export_textline_images_and_text=False, do_not_mask_with_textline_contour=False, draw_texts_on_image=False, @@ -4895,6 +4896,10 @@ class Eynollah_ocr: self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) self.model_ocr.to(self.device) + if not batch_size: + self.b_s = 2 + else: + self.b_s = int(batch_size) else: self.model_ocr_dir = dir_models + "/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" @@ -4903,6 +4908,10 @@ class Eynollah_ocr: self.prediction_model = tf.keras.models.Model( model_ocr.get_layer(name = "image").input, model_ocr.get_layer(name = "dense2").output) + if not batch_size: + self.b_s = 8 + else: + self.b_s = int(batch_size) with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: @@ -4918,6 +4927,7 @@ class Eynollah_ocr: self.num_to_char = StringLookup( vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True ) + def decode_batch_predictions(self, pred, max_len = 128): # input_len is the product of the batch size and the @@ -5073,10 +5083,9 @@ class Eynollah_ocr: ls_imgs = os.listdir(self.dir_in) if self.tr_ocr: - b_s = 2 + tr_ocr_input_height_and_width = 384 for ind_img in ls_imgs: - t0 = time.time() - file_name = ind_img.split('.')[0] + file_name = Path(ind_img).stem dir_img = os.path.join(self.dir_in, ind_img) dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') @@ -5131,15 +5140,15 @@ class Eynollah_ocr: img_crop[mask_poly==0] = 255 if h2w_ratio > 0.1: - cropped_lines.append(img_crop) + cropped_lines.append(resize_image(img_crop, tr_ocr_input_height_and_width, tr_ocr_input_height_and_width) ) cropped_lines_meging_indexing.append(0) else: splited_images, _ = self.return_textlines_split_if_needed(img_crop, None) #print(splited_images) if splited_images: - cropped_lines.append(splited_images[0]) + cropped_lines.append(resize_image(splited_images[0], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) cropped_lines_meging_indexing.append(1) - cropped_lines.append(splited_images[1]) + cropped_lines.append(resize_image(splited_images[1], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) cropped_lines_meging_indexing.append(-1) else: cropped_lines.append(img_crop) @@ -5148,21 +5157,24 @@ class Eynollah_ocr: extracted_texts = [] - n_iterations = math.ceil(len(cropped_lines) / b_s) + n_iterations = math.ceil(len(cropped_lines) / self.b_s) for i in range(n_iterations): if i==(n_iterations-1): - n_start = i*b_s + n_start = i*self.b_s imgs = cropped_lines[n_start:] else: - n_start = i*b_s - n_end = (i+1)*b_s + n_start = i*self.b_s + n_end = (i+1)*self.b_s imgs = cropped_lines[n_start:n_end] pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) extracted_texts = extracted_texts + generated_text_merged + + del cropped_lines + gc.collect() extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] @@ -5241,14 +5253,12 @@ class Eynollah_ocr: padding_token = 299 image_width = 512#max_len * 4 image_height = 32 - b_s = 8 img_size=(image_width, image_height) for ind_img in ls_imgs: - t0 = time.time() - file_name = ind_img.split('.')[0] + file_name = Path(ind_img).stem dir_img = os.path.join(self.dir_in, ind_img) dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') @@ -5368,11 +5378,11 @@ class Eynollah_ocr: if not self.export_textline_images_and_text: extracted_texts = [] - n_iterations = math.ceil(len(cropped_lines) / b_s) + n_iterations = math.ceil(len(cropped_lines) / self.b_s) for i in range(n_iterations): if i==(n_iterations-1): - n_start = i*b_s + n_start = i*self.b_s imgs = cropped_lines[n_start:] imgs = np.array(imgs) imgs = imgs.reshape(imgs.shape[0], image_height, image_width, 3) @@ -5381,14 +5391,14 @@ class Eynollah_ocr: imgs_bin = np.array(imgs_bin) imgs_bin = imgs_bin.reshape(imgs_bin.shape[0], image_height, image_width, 3) else: - n_start = i*b_s - n_end = (i+1)*b_s + n_start = i*self.b_s + n_end = (i+1)*self.b_s imgs = cropped_lines[n_start:n_end] - imgs = np.array(imgs).reshape(b_s, image_height, image_width, 3) + imgs = np.array(imgs).reshape(self.b_s, image_height, image_width, 3) if self.prediction_with_both_of_rgb_and_bin: imgs_bin = cropped_lines_bin[n_start:n_end] - imgs_bin = np.array(imgs_bin).reshape(b_s, image_height, image_width, 3) + imgs_bin = np.array(imgs_bin).reshape(self.b_s, image_height, image_width, 3) preds = self.prediction_model.predict(imgs, verbose=0) @@ -5402,6 +5412,11 @@ class Eynollah_ocr: pred_texts_ib = pred_texts[ib].replace("[UNK]", "") extracted_texts.append(pred_texts_ib) + del cropped_lines + if self.prediction_with_both_of_rgb_and_bin: + del cropped_lines_bin + gc.collect() + extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] From 48e8dd4ab3ac9238fff0f1a7147ecfff9dab23e9 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 2 May 2025 12:57:26 +0200 Subject: [PATCH 119/374] machine based model name changed to public one --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 62026bf..cc1f766 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -259,7 +259,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_mb_ro_aug_2"#"/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" From 89aa5450491d84d816e68de5603018c0b820eedb Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sat, 3 May 2025 02:59:16 +0200 Subject: [PATCH 120/374] let to add dataset abbrevation to extracted textline images and text --- src/eynollah/cli.py | 17 +++++++- src/eynollah/eynollah.py | 91 ++++++++++++++++++++++++---------------- 2 files changed, 71 insertions(+), 37 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 56d5d7e..7d08ac8 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -342,7 +342,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ "-m", help="directory of models", type=click.Path(exists=True, file_okay=False), - required=True, ) @click.option( "--tr_ocr", @@ -379,6 +378,11 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ "-bs", help="number of inference batch size. Default b_s for trocr and cnn_rnn models are 2 and 8 respectively", ) +@click.option( + "--dataset_abbrevation", + "-ds_pref", + help="in the case of extracting textline and text from a xml GT file user can add an abbrevation of dataset name to generated dataset", +) @click.option( "--log_level", "-l", @@ -386,10 +390,18 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, log_level): +def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) + assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text -etit can not be set alongside transformer ocr -tr_ocr" + assert not export_textline_images_and_text or not model, "Exporting textline and text -etit can not be set alongside model -m" + assert not export_textline_images_and_text or not batch_size, "Exporting textline and text -etit can not be set alongside batch size -bs" + assert not export_textline_images_and_text or not dir_in_bin, "Exporting textline and text -etit can not be set alongside directory of bin images -dib" + assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit" + assert not export_textline_images_and_text or not draw_texts_on_image, "Exporting textline and text -etit can not be set alongside draw text on image -dtoi" + assert not export_textline_images_and_text or not prediction_with_both_of_rgb_and_bin, "Exporting textline and text -etit can not be set alongside prediction with both rgb and bin -brb" + eynollah_ocr = Eynollah_ocr( dir_xmls=dir_xmls, dir_out_image_text=dir_out_image_text, @@ -403,6 +415,7 @@ def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, ex draw_texts_on_image=draw_texts_on_image, prediction_with_both_of_rgb_and_bin=prediction_with_both_of_rgb_and_bin, batch_size=batch_size, + pref_of_dataset=dataset_abbrevation, ) eynollah_ocr.run() diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index cc1f766..0b15573 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4877,6 +4877,7 @@ class Eynollah_ocr: do_not_mask_with_textline_contour=False, draw_texts_on_image=False, prediction_with_both_of_rgb_and_bin=False, + pref_of_dataset = None, logger=None, ): self.dir_in = dir_in @@ -4890,43 +4891,45 @@ class Eynollah_ocr: self.draw_texts_on_image = draw_texts_on_image self.dir_out_image_text = dir_out_image_text self.prediction_with_both_of_rgb_and_bin = prediction_with_both_of_rgb_and_bin - if tr_ocr: - self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" - self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) - self.model_ocr.to(self.device) - if not batch_size: - self.b_s = 2 + self.pref_of_dataset = pref_of_dataset + if not export_textline_images_and_text: + if tr_ocr: + self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") + self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" + self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) + self.model_ocr.to(self.device) + if not batch_size: + self.b_s = 2 + else: + self.b_s = int(batch_size) + else: - self.b_s = int(batch_size) - - else: - self.model_ocr_dir = dir_models + "/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" - model_ocr = load_model(self.model_ocr_dir , compile=False) - - self.prediction_model = tf.keras.models.Model( - model_ocr.get_layer(name = "image").input, - model_ocr.get_layer(name = "dense2").output) - if not batch_size: - self.b_s = 8 - else: - self.b_s = int(batch_size) - + self.model_ocr_dir = dir_models + "/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + model_ocr = load_model(self.model_ocr_dir , compile=False) - with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: - characters = json.load(config_file) + self.prediction_model = tf.keras.models.Model( + model_ocr.get_layer(name = "image").input, + model_ocr.get_layer(name = "dense2").output) + if not batch_size: + self.b_s = 8 + else: + self.b_s = int(batch_size) - - AUTOTUNE = tf.data.AUTOTUNE + + with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: + characters = json.load(config_file) - # Mapping characters to integers. - char_to_num = StringLookup(vocabulary=list(characters), mask_token=None) + + AUTOTUNE = tf.data.AUTOTUNE - # Mapping integers back to original characters. - self.num_to_char = StringLookup( - vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True - ) + # Mapping characters to integers. + char_to_num = StringLookup(vocabulary=list(characters), mask_token=None) + + # Mapping integers back to original characters. + self.num_to_char = StringLookup( + vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True + ) def decode_batch_predictions(self, pred, max_len = 128): @@ -5365,10 +5368,28 @@ class Eynollah_ocr: if cheild_text.tag.endswith("Unicode"): textline_text = cheild_text.text if textline_text: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.txt'), 'w') as text_file: - text_file.write(textline_text) + if self.do_not_mask_with_textline_contour: + if self.pref_of_dataset: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.txt'), 'w') as text_file: + text_file.write(textline_text) - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.png'), img_crop ) + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.png'), img_crop ) + else: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.txt'), 'w') as text_file: + text_file.write(textline_text) + + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.png'), img_crop ) + else: + if self.pref_of_dataset: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.txt'), 'w') as text_file: + text_file.write(textline_text) + + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.png'), img_crop ) + else: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.txt'), 'w') as text_file: + text_file.write(textline_text) + + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.png'), img_crop ) indexer_textlines+=1 From 3b123b039c432145359f7b6a3b0d45c8669df791 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sat, 3 May 2025 19:25:32 +0200 Subject: [PATCH 121/374] adding min_early parameter for generating training dataset for machine based reading order model --- train/generate_gt_for_training.py | 64 +++++++++++++++++++++++-------- train/gt_gen_utils.py | 13 ++++++- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 9869bfa..77e9238 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -147,11 +147,20 @@ def image_enhancement(dir_imgs, dir_out_images, dir_out_labels, scales): help="min area size of regions considered for reading order training.", ) -def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width, min_area_size): +@click.option( + "--min_area_early", + "-min_early", + help="If you have already generated a training dataset using a specific minimum area value and now wish to create a dataset with a smaller minimum area value, you can avoid regenerating the previous dataset by providing the earlier minimum area value. This will ensure that only the missing data is generated.", +) + +def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width, min_area_size, min_area_early): xml_files_ind = os.listdir(dir_xml) input_height = int(input_height) input_width = int(input_width) min_area = float(min_area_size) + if min_area_early: + min_area_early = float(min_area_early) + indexer_start= 0#55166 max_area = 1 @@ -181,7 +190,8 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i texts_corr_order_index_int = [int(x) for x in texts_corr_order_index] - co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area) + co_text_all, texts_corr_order_index_int, regions_ar_less_than_early_min = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, min_area, min_area_early) + arg_array = np.array(range(len(texts_corr_order_index_int))) @@ -195,25 +205,49 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i labels_con[:,:,i] = img_label[:,:,0] + labels_con = resize_image(labels_con, input_height, input_width) + img_poly = resize_image(img_poly, input_height, input_width) + + for i in range(len(texts_corr_order_index_int)): for j in range(len(texts_corr_order_index_int)): if i!=j: - input_multi_visual_modal = np.zeros((input_height,input_width,3)).astype(np.int8) - final_f_name = f_name+'_'+str(indexer+indexer_start) - order_class_condition = texts_corr_order_index_int[i]-texts_corr_order_index_int[j] - if order_class_condition<0: - class_type = 1 + if regions_ar_less_than_early_min: + if regions_ar_less_than_early_min[i]==1: + input_multi_visual_modal = np.zeros((input_height,input_width,3)).astype(np.int8) + final_f_name = f_name+'_'+str(indexer+indexer_start) + order_class_condition = texts_corr_order_index_int[i]-texts_corr_order_index_int[j] + if order_class_condition<0: + class_type = 1 + else: + class_type = 0 + + input_multi_visual_modal[:,:,0] = labels_con[:,:,i] + input_multi_visual_modal[:,:,1] = img_poly[:,:,0] + input_multi_visual_modal[:,:,2] = labels_con[:,:,j] + + np.save(os.path.join(dir_out_classes,final_f_name+'_missed.npy' ), class_type) + + cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'_missed.png' ), input_multi_visual_modal) + indexer = indexer+1 + else: - class_type = 0 + input_multi_visual_modal = np.zeros((input_height,input_width,3)).astype(np.int8) + final_f_name = f_name+'_'+str(indexer+indexer_start) + order_class_condition = texts_corr_order_index_int[i]-texts_corr_order_index_int[j] + if order_class_condition<0: + class_type = 1 + else: + class_type = 0 - input_multi_visual_modal[:,:,0] = resize_image(labels_con[:,:,i], input_height, input_width) - input_multi_visual_modal[:,:,1] = resize_image(img_poly[:,:,0], input_height, input_width) - input_multi_visual_modal[:,:,2] = resize_image(labels_con[:,:,j], input_height, input_width) + input_multi_visual_modal[:,:,0] = labels_con[:,:,i] + input_multi_visual_modal[:,:,1] = img_poly[:,:,0] + input_multi_visual_modal[:,:,2] = labels_con[:,:,j] - np.save(os.path.join(dir_out_classes,final_f_name+'.npy' ), class_type) - - cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'.png' ), input_multi_visual_modal) - indexer = indexer+1 + np.save(os.path.join(dir_out_classes,final_f_name+'.npy' ), class_type) + + cv2.imwrite(os.path.join(dir_out_modal_image,final_f_name+'.png' ), input_multi_visual_modal) + indexer = indexer+1 @main.command() diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 753abf2..10183d6 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -51,9 +51,10 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m jv += 1 return found_polygons_early -def filter_contours_area_of_image(image, contours, order_index, max_area, min_area): +def filter_contours_area_of_image(image, contours, order_index, max_area, min_area, min_early): found_polygons_early = list() order_index_filtered = list() + regions_ar_less_than_early_min = list() #jv = 0 for jv, c in enumerate(contours): if len(np.shape(c)) == 3: @@ -68,8 +69,16 @@ def filter_contours_area_of_image(image, contours, order_index, max_area, min_ar if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.uint)) order_index_filtered.append(order_index[jv]) + if min_early: + if area < min_early * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : + regions_ar_less_than_early_min.append(1) + else: + regions_ar_less_than_early_min.append(0) + else: + regions_ar_less_than_early_min = None + #jv += 1 - return found_polygons_early, order_index_filtered + return found_polygons_early, order_index_filtered, regions_ar_less_than_early_min def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): From 5694d971c5c068413b0a35db1aceabd50963107d Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 5 May 2025 15:39:05 +0200 Subject: [PATCH 122/374] saving model by steps is added to reading order and pixel wise segmentation use cases training --- train/train.py | 60 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 9 deletions(-) diff --git a/train/train.py b/train/train.py index 5dee567..df600a8 100644 --- a/train/train.py +++ b/train/train.py @@ -13,8 +13,29 @@ from tensorflow.keras.models import load_model from tqdm import tqdm import json from sklearn.metrics import f1_score +from tensorflow.keras.callbacks import Callback +class SaveWeightsAfterSteps(Callback): + def __init__(self, save_interval, save_path, _config): + super(SaveWeightsAfterSteps, self).__init__() + self.save_interval = save_interval + self.save_path = save_path + self.step_count = 0 + def on_train_batch_end(self, batch, logs=None): + self.step_count += 1 + + if self.step_count % self.save_interval ==0: + save_file = f"{self.save_path}/model_step_{self.step_count}" + #os.system('mkdir '+save_file) + + self.model.save(save_file) + + with open(os.path.join(os.path.join(save_path, "model_step_{self.step_count}"),"config.json"), "w") as fp: + json.dump(_config, fp) # encode dict into JSON + print(f"saved model as steps {self.step_count} to {save_file}") + + def configuration(): config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True @@ -93,7 +114,7 @@ def config_params(): f1_threshold_classification = None # This threshold is used to consider models with an evaluation f1 scores bigger than it. The selected model weights undergo a weights ensembling. And avreage ensembled model will be written to output. classification_classes_name = None # Dictionary of classification classes names. backbone_type = None # As backbone we have 2 types of backbones. A vision transformer alongside a CNN and we call it "transformer" and only CNN called "nontransformer" - + save_interval = None dir_img_bin = None number_of_backgrounds_per_image = 1 dir_rgb_backgrounds = None @@ -112,7 +133,7 @@ def run(_config, n_classes, n_epochs, input_height, thetha, scaling_flip, continue_training, transformer_projection_dim, transformer_mlp_head_units, transformer_layers, transformer_num_heads, transformer_cnn_first, transformer_patchsize_x, transformer_patchsize_y, - transformer_num_patches_xy, backbone_type, flip_index, dir_eval, dir_output, + transformer_num_patches_xy, backbone_type, save_interval, flip_index, dir_eval, dir_output, pretraining, learning_rate, task, f1_threshold_classification, classification_classes_name, dir_img_bin, number_of_backgrounds_per_image,dir_rgb_backgrounds, dir_rgb_foregrounds): if dir_rgb_backgrounds: @@ -299,13 +320,27 @@ def run(_config, n_classes, n_epochs, input_height, ##img_validation_patches = os.listdir(dir_flow_eval_imgs) ##score_best=[] ##score_best.append(0) + + if save_interval: + save_weights_callback = SaveWeightsAfterSteps(save_interval, dir_output, _config) + + for i in tqdm(range(index_start, n_epochs + index_start)): - model.fit( - train_gen, - steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, - validation_data=val_gen, - validation_steps=1, - epochs=1) + if save_interval: + model.fit( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, + validation_data=val_gen, + validation_steps=1, + epochs=1, callbacks=[save_weights_callback]) + else: + model.fit( + train_gen, + steps_per_epoch=int(len(os.listdir(dir_flow_train_imgs)) / n_batch) - 1, + validation_data=val_gen, + validation_steps=1, + epochs=1) + model.save(os.path.join(dir_output,'model_'+str(i))) with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: @@ -392,8 +427,15 @@ def run(_config, n_classes, n_epochs, input_height, opt_adam = tf.keras.optimizers.Adam(learning_rate=0.0001) model.compile(loss="binary_crossentropy", optimizer = opt_adam,metrics=['accuracy']) + + if save_interval: + save_weights_callback = SaveWeightsAfterSteps(save_interval, dir_output, _config) + for i in range(n_epochs): - history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes, thetha, augmentation), steps_per_epoch=num_rows / n_batch, verbose=1) + if save_interval: + history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes, thetha, augmentation), steps_per_epoch=num_rows / n_batch, verbose=1, callbacks=[save_weights_callback]) + else: + history = model.fit(generate_arrays_from_folder_reading_order(dir_flow_train_labels, dir_flow_train_imgs, n_batch, input_height, input_width, n_classes, thetha, augmentation), steps_per_epoch=num_rows / n_batch, verbose=1) model.save( os.path.join(dir_output,'model_'+str(i+indexer_start) )) with open(os.path.join(os.path.join(dir_output,'model_'+str(i)),"config.json"), "w") as fp: From 92954b1b7b7363f8cdae91500cf0e729c2eebc62 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 5 May 2025 16:13:38 +0200 Subject: [PATCH 123/374] resolving issued with saving model by steps --- train/train.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/train/train.py b/train/train.py index df600a8..f6a4f47 100644 --- a/train/train.py +++ b/train/train.py @@ -21,6 +21,7 @@ class SaveWeightsAfterSteps(Callback): self.save_interval = save_interval self.save_path = save_path self.step_count = 0 + self._config = _config def on_train_batch_end(self, batch, logs=None): self.step_count += 1 @@ -31,8 +32,8 @@ class SaveWeightsAfterSteps(Callback): self.model.save(save_file) - with open(os.path.join(os.path.join(save_path, "model_step_{self.step_count}"),"config.json"), "w") as fp: - json.dump(_config, fp) # encode dict into JSON + with open(os.path.join(os.path.join(self.save_path, f"model_step_{self.step_count}"),"config.json"), "w") as fp: + json.dump(self._config, fp) # encode dict into JSON print(f"saved model as steps {self.step_count} to {save_file}") From 83211ae684513ef7f50ee88e0f641702441cde1f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 7 May 2025 12:33:03 +0200 Subject: [PATCH 124/374] In the case of skip_layout_and_reading_order, the confidence value was not set correctly, leading to an error while writing to the XML file. --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 022cf0a..ec8d887 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4328,7 +4328,7 @@ class Eynollah: polygons_lines_xml = [] contours_tables = [] ocr_all_textlines = None - conf_contours_textregions =None + conf_contours_textregions = [0] pcgts = self.writer.build_pagexml_no_full_layout( cont_page, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, From 21ec4fbfb538b40f0d06f55bf8c92f4ca2ebf10c Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 7 May 2025 14:04:01 +0200 Subject: [PATCH 125/374] The text region coordinates are now correctly written into the XML output when using the skip layout and reading order option --- src/eynollah/eynollah.py | 2 +- src/eynollah/writer.py | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index ec8d887..6da003b 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4333,7 +4333,7 @@ class Eynollah: cont_page, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions, self.skip_layout_and_reading_order) return pcgts #print("text region early -1 in %.1fs", time.time() - t0) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 92e353f..e589fd4 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -168,7 +168,7 @@ class EynollahXmlWriter(): with open(self.output_filename, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines, conf_contours_textregion): + def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines, conf_contours_textregion, skip_layout_reading_order=False): self.logger.debug('enter build_pagexml_no_full_layout') # create the file structure @@ -184,7 +184,7 @@ class EynollahXmlWriter(): for mm in range(len(found_polygons_text_region)): textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord), conf=conf_contours_textregion[mm]), + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord, skip_layout_reading_order), conf=conf_contours_textregion[mm]), ) #textregion.set_conf(conf_contours_textregion[mm]) page.add_TextRegion(textregion) @@ -303,18 +303,28 @@ class EynollahXmlWriter(): return pcgts - def calculate_polygon_coords(self, contour, page_coord): + def calculate_polygon_coords(self, contour, page_coord, skip_layout_reading_order=False): self.logger.debug('enter calculate_polygon_coords') coords = '' for value_bbox in contour: - if len(value_bbox) == 2: - coords += str(int((value_bbox[0] + page_coord[2]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[1] + page_coord[0]) / self.scale_y)) + if skip_layout_reading_order: + if len(value_bbox) == 2: + coords += str(int((value_bbox[0]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[1]) / self.scale_y)) + else: + coords += str(int((value_bbox[0][0]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[0][1]) / self.scale_y)) else: - coords += str(int((value_bbox[0][0] + page_coord[2]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[0][1] + page_coord[0]) / self.scale_y)) + if len(value_bbox) == 2: + coords += str(int((value_bbox[0] + page_coord[2]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[1] + page_coord[0]) / self.scale_y)) + else: + coords += str(int((value_bbox[0][0] + page_coord[2]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[0][1] + page_coord[0]) / self.scale_y)) coords=coords + ' ' return coords[:-1] From 6fa766d6a566fa4660c0c7424ddebb85f1a0d0c7 Mon Sep 17 00:00:00 2001 From: johnlockejrr <16368414+johnlockejrr@users.noreply.github.com> Date: Sun, 11 May 2025 05:31:34 -0700 Subject: [PATCH 126/374] Update utils.py --- train/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train/utils.py b/train/utils.py index 3d42b64..cba20c2 100644 --- a/train/utils.py +++ b/train/utils.py @@ -667,7 +667,7 @@ def provide_patches(imgs_list_train, segs_list_train, dir_img, dir_seg, dir_flow indexer = 0 for im, seg_i in tqdm(zip(imgs_list_train, segs_list_train)): - img_name = im.split('.')[0] + img_name = os.path.splitext(im)[0] if task == "segmentation" or task == "binarization": dir_of_label_file = os.path.join(dir_seg, img_name + '.png') elif task=="enhancement": From 3a9fc0efde07a4890995adbfefc8d135e9278747 Mon Sep 17 00:00:00 2001 From: johnlockejrr <16368414+johnlockejrr@users.noreply.github.com> Date: Sun, 11 May 2025 06:09:17 -0700 Subject: [PATCH 127/374] Update utils.py Changed unsafe basename extraction: `file_name = i.split('.')[0]` to `file_name = os.path.splitext(i)[0]` and `filename = n[i].split('.')[0]` to `filename = os.path.splitext(n[i])[0]` because `"Vat.sam.2_206.jpg` -> `Vat` instead of `"Vat.sam.2_206` --- train/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train/utils.py b/train/utils.py index cba20c2..bbe21d1 100644 --- a/train/utils.py +++ b/train/utils.py @@ -374,7 +374,7 @@ def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batch batchcount = 0 while True: for i in all_labels_files: - file_name = i.split('.')[0] + file_name = os.path.splitext(i)[0] img = cv2.imread(os.path.join(modal_dir,file_name+'.png')) label_class = int( np.load(os.path.join(classes_file_dir,i)) ) @@ -401,7 +401,7 @@ def data_gen(img_folder, mask_folder, batch_size, input_height, input_width, n_c for i in range(c, c + batch_size): # initially from 0 to 16, c = 0. try: - filename = n[i].split('.')[0] + filename = os.path.splitext(n[i])[0] train_img = cv2.imread(img_folder + '/' + n[i]) / 255. train_img = cv2.resize(train_img, (input_width, input_height), From c12b09a8686476291aa58231445cc535bb13b888 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 12 May 2025 00:10:18 +0200 Subject: [PATCH 128/374] I have tried to address the issues #163 and #161 . The changes have also improved marginal detection and enhanced the isolation of headers. --- requirements.txt | 1 + src/eynollah/cli.py | 14 +- src/eynollah/eynollah.py | 294 ++++++++++++++++++++++++++++++++++----- 3 files changed, 275 insertions(+), 34 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9ed0584..aeffd47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ numpy <1.24.0 scikit-learn >= 0.23.2 tensorflow < 2.13 numba <= 0.58.1 +scikit-image loky diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 7d08ac8..99961c9 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -235,6 +235,16 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) "-ncl", help="upper limit of columns in document image", ) +@click.option( + "--threshold_art_class_layout", + "-tharl", + help="threshold of artifical class in the case of layout detection", +) +@click.option( + "--threshold_art_class_textline", + "-thart", + help="threshold of artifical class in the case of textline detection", +) @click.option( "--skip_layout_and_reading_order", "-slro/-noslro", @@ -248,7 +258,7 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) help="Override log level globally to this", ) -def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, num_col_upper, num_col_lower, skip_layout_and_reading_order, ignore_page_extraction, log_level): +def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, num_col_upper, num_col_lower, threshold_art_class_textline, threshold_art_class_layout, skip_layout_and_reading_order, ignore_page_extraction, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) @@ -298,6 +308,8 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ num_col_upper=num_col_upper, num_col_lower=num_col_lower, skip_layout_and_reading_order=skip_layout_and_reading_order, + threshold_art_class_textline=threshold_art_class_textline, + threshold_art_class_layout=threshold_art_class_layout, ) if dir_in: eynollah.run(dir_in=dir_in, overwrite=overwrite) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0b15573..0c7c5d2 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -30,7 +30,7 @@ import numpy as np from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d from numba import cuda - +from skimage.morphology import skeletonize from ocrd import OcrdPage from ocrd_utils import getLogger, tf_disable_interactive_logs @@ -200,6 +200,8 @@ class Eynollah: do_ocr : bool = False, num_col_upper : Optional[int] = None, num_col_lower : Optional[int] = None, + threshold_art_class_layout: Optional[float] = None, + threshold_art_class_textline: Optional[float] = None, skip_layout_and_reading_order : bool = False, logger : Optional[Logger] = None, ): @@ -237,6 +239,17 @@ class Eynollah: self.num_col_lower = int(num_col_lower) else: self.num_col_lower = num_col_lower + + if threshold_art_class_layout: + self.threshold_art_class_layout = float(threshold_art_class_layout) + else: + self.threshold_art_class_layout = 0.1 + + if threshold_art_class_textline: + self.threshold_art_class_textline = float(threshold_art_class_textline) + else: + self.threshold_art_class_textline = 0.1 + self.logger = logger if logger else getLogger('eynollah') # for parallelization of CPU-intensive tasks: self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) @@ -784,7 +797,7 @@ class Eynollah: self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False): + thresholding_for_artificial_class_in_light_version=False, threshold_art_class_textline=0.1): self.logger.debug("enter do_prediction") img_height_model = model.layers[-1].output_shape[1] @@ -802,10 +815,13 @@ class Eynollah: if thresholding_for_artificial_class_in_light_version: seg_art = label_p_pred[0,:,:,2] - seg_art[seg_art<0.2] = 0 + seg_art[seg_art0] =1 + + skeleton_art = skeletonize(seg_art) + skeleton_art = skeleton_art*1 - seg[seg_art==1]=2 + seg[skeleton_art==1]=2 seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) return prediction_true @@ -896,14 +912,17 @@ class Eynollah: if thresholding_for_artificial_class_in_light_version: seg_art = label_p_pred[:,:,:,2] - seg_art[seg_art<0.2] = 0 + seg_art[seg_art0] =1 - seg[seg_art==1]=2 + ##seg[seg_art==1]=2 indexer_inside_batch = 0 for i_batch, j_batch in zip(list_i_s, list_j_s): seg_in = seg[indexer_inside_batch] + + if thresholding_for_artificial_class_in_light_version: + seg_in_art = seg_art[indexer_inside_batch] index_y_u_in = list_y_u[indexer_inside_batch] index_y_d_in = list_y_d[indexer_inside_batch] @@ -917,54 +936,107 @@ class Eynollah: seg_in[0:-margin or None, 0:-margin or None, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[0:-margin or None, + 0:-margin or None] + elif i_batch == nxf - 1 and j_batch == nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + margin:index_x_u_in - 0] = \ seg_in[margin:, margin:, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[margin:, + margin:] + elif i_batch == 0 and j_batch == nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + 0:index_x_u_in - margin] = \ seg_in[margin:, 0:-margin or None, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[margin:, + 0:-margin or None] + elif i_batch == nxf - 1 and j_batch == 0: prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - 0] = \ seg_in[0:-margin or None, margin:, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[0:-margin or None, + margin:] + elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + 0:index_x_u_in - margin] = \ seg_in[margin:-margin or None, 0:-margin or None, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[margin:-margin or None, + 0:-margin or None] + elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - 0] = \ seg_in[margin:-margin or None, margin:, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[margin:-margin or None, + margin:] + elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - margin] = \ seg_in[0:-margin or None, margin:-margin or None, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[0:-margin or None, + margin:-margin or None] + elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + margin:index_x_u_in - margin] = \ seg_in[margin:, margin:-margin or None, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[margin:, + margin:-margin or None] + else: prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - margin] = \ seg_in[margin:-margin or None, margin:-margin or None, np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[margin:-margin or None, + margin:-margin or None] indexer_inside_batch += 1 @@ -979,6 +1051,19 @@ class Eynollah: img_patch[:] = 0 prediction_true = prediction_true.astype(np.uint8) + + if thresholding_for_artificial_class_in_light_version: + kernel_min = np.ones((3, 3), np.uint8) + prediction_true[:,:,0][prediction_true[:,:,0]==2] = 0 + + skeleton_art = skeletonize(prediction_true[:,:,1]) + skeleton_art = skeleton_art*1 + + skeleton_art = skeleton_art.astype('uint8') + + skeleton_art = cv2.dilate(skeleton_art, kernel_min, iterations=1) + + prediction_true[:,:,0][skeleton_art==1]=2 #del model gc.collect() return prediction_true @@ -1117,7 +1202,7 @@ class Eynollah: self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False): + thresholding_for_artificial_class_in_light_version=False, threshold_art_class_textline=0.1, threshold_art_class_layout=0.1): self.logger.debug("enter do_prediction_new_concept") img_height_model = model.layers[-1].output_shape[1] @@ -1132,19 +1217,28 @@ class Eynollah: label_p_pred = model.predict(img[np.newaxis], verbose=0) seg = np.argmax(label_p_pred, axis=3)[0] - if thresholding_for_artificial_class_in_light_version: - #seg_text = label_p_pred[0,:,:,1] - #seg_text[seg_text<0.2] =0 - #seg_text[seg_text>0] =1 - #seg[seg_text==1]=1 - - seg_art = label_p_pred[0,:,:,4] - seg_art[seg_art<0.2] =0 - seg_art[seg_art>0] =1 - seg[seg_art==1]=4 - seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) + + if thresholding_for_artificial_class_in_light_version: + kernel_min = np.ones((3, 3), np.uint8) + seg_art = label_p_pred[0,:,:,4] + seg_art[seg_art0] =1 + #seg[seg_art==1]=4 + seg_art = resize_image(seg_art, img_h_page, img_w_page).astype(np.uint8) + + prediction_true[:,:,0][prediction_true[:,:,0]==4] = 0 + + skeleton_art = skeletonize(seg_art) + skeleton_art = skeleton_art*1 + + skeleton_art = skeleton_art.astype('uint8') + + skeleton_art = cv2.dilate(skeleton_art, kernel_min, iterations=1) + + prediction_true[:,:,0][skeleton_art==1] = 4 + return prediction_true , resize_image(label_p_pred[0, :, :, 1] , img_h_page, img_w_page) if img.shape[0] < img_height_model: @@ -1217,26 +1311,29 @@ class Eynollah: if thresholding_for_some_classes_in_light_version: seg_art = label_p_pred[:,:,:,4] - seg_art[seg_art<0.2] =0 + seg_art[seg_art0] =1 seg_line = label_p_pred[:,:,:,3] seg_line[seg_line>0.5] =1#seg_line[seg_line>0.1] =1 seg_line[seg_line<1] =0 - seg[seg_art==1]=4 + ##seg[seg_art==1]=4 seg[(seg_line==1) & (seg==0)]=3 if thresholding_for_artificial_class_in_light_version: seg_art = label_p_pred[:,:,:,2] - seg_art[seg_art<0.2] = 0 + seg_art[seg_art0] =1 - seg[seg_art==1]=2 + ##seg[seg_art==1]=2 indexer_inside_batch = 0 for i_batch, j_batch in zip(list_i_s, list_j_s): seg_in = seg[indexer_inside_batch] + + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + seg_in_art = seg_art[indexer_inside_batch] index_y_u_in = list_y_u[indexer_inside_batch] index_y_d_in = list_y_d[indexer_inside_batch] @@ -1255,6 +1352,12 @@ class Eynollah: label_p_pred[0, 0:-margin or None, 0:-margin or None, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[0:-margin or None, + 0:-margin or None] + elif i_batch == nxf - 1 and j_batch == nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + margin:index_x_u_in - 0] = \ @@ -1266,6 +1369,12 @@ class Eynollah: label_p_pred[0, margin:, margin:, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[margin:, + margin:] + elif i_batch == 0 and j_batch == nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + 0:index_x_u_in - margin] = \ @@ -1277,6 +1386,13 @@ class Eynollah: label_p_pred[0, margin:, 0:-margin or None, 1] + + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[margin:, + 0:-margin or None] + elif i_batch == nxf - 1 and j_batch == 0: prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - 0] = \ @@ -1288,6 +1404,12 @@ class Eynollah: label_p_pred[0, 0:-margin or None, margin:, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[0:-margin or None, + margin:] + elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + 0:index_x_u_in - margin] = \ @@ -1299,6 +1421,11 @@ class Eynollah: label_p_pred[0, margin:-margin or None, 0:-margin or None, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[margin:-margin or None, + 0:-margin or None] elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - 0] = \ @@ -1310,6 +1437,11 @@ class Eynollah: label_p_pred[0, margin:-margin or None, margin:, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[margin:-margin or None, + margin:] elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - margin] = \ @@ -1321,6 +1453,11 @@ class Eynollah: label_p_pred[0, 0:-margin or None, margin:-margin or None, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[0:-margin or None, + margin:-margin or None] elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + margin:index_x_u_in - margin] = \ @@ -1332,6 +1469,11 @@ class Eynollah: label_p_pred[0, margin:, margin:-margin or None, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[margin:, + margin:-margin or None] else: prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - margin] = \ @@ -1343,6 +1485,11 @@ class Eynollah: label_p_pred[0, margin:-margin or None, margin:-margin or None, 1] + if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[margin:-margin or None, + margin:-margin or None] indexer_inside_batch += 1 list_i_s = [] @@ -1356,6 +1503,32 @@ class Eynollah: img_patch[:] = 0 prediction_true = prediction_true.astype(np.uint8) + + if thresholding_for_artificial_class_in_light_version: + kernel_min = np.ones((3, 3), np.uint8) + prediction_true[:,:,0][prediction_true[:,:,0]==2] = 0 + + skeleton_art = skeletonize(prediction_true[:,:,1]) + skeleton_art = skeleton_art*1 + + skeleton_art = skeleton_art.astype('uint8') + + skeleton_art = cv2.dilate(skeleton_art, kernel_min, iterations=1) + + prediction_true[:,:,0][skeleton_art==1]=2 + + if thresholding_for_some_classes_in_light_version: + kernel_min = np.ones((3, 3), np.uint8) + prediction_true[:,:,0][prediction_true[:,:,0]==4] = 0 + + skeleton_art = skeletonize(prediction_true[:,:,1]) + skeleton_art = skeleton_art*1 + + skeleton_art = skeleton_art.astype('uint8') + + skeleton_art = cv2.dilate(skeleton_art, kernel_min, iterations=1) + + prediction_true[:,:,0][skeleton_art==1]=4 gc.collect() return prediction_true, confidence_matrix @@ -1608,7 +1781,7 @@ class Eynollah: prediction_textline = self.do_prediction( use_patches, img, self.model_textline, marginal_of_patch_percent=0.15, n_batch_inference=3, - thresholding_for_artificial_class_in_light_version=self.textline_light) + thresholding_for_artificial_class_in_light_version=self.textline_light, threshold_art_class_textline=self.threshold_art_class_textline) #if not self.textline_light: #if num_col_classifier==1: #prediction_textline_nopatch = self.do_prediction(False, img, self.model_textline) @@ -1622,7 +1795,55 @@ class Eynollah: textline_mask_tot_ea_art = textline_mask_tot_ea_art.astype('uint8') #textline_mask_tot_ea_art = cv2.dilate(textline_mask_tot_ea_art, KERNEL, iterations=1) prediction_textline[:,:][textline_mask_tot_ea_art[:,:]==1]=2 + """ + else: + textline_mask_tot_ea_art = textline_mask_tot_ea_art.astype('uint8') + hor_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (8, 1)) + + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) + ##cv2.imwrite('textline_mask_tot_ea_art.png', textline_mask_tot_ea_art) + textline_mask_tot_ea_art = cv2.dilate(textline_mask_tot_ea_art, hor_kernel, iterations=1) + + ###cv2.imwrite('dil_textline_mask_tot_ea_art.png', dil_textline_mask_tot_ea_art) + + textline_mask_tot_ea_art = textline_mask_tot_ea_art.astype('uint8') + + #print(np.shape(dil_textline_mask_tot_ea_art), np.unique(dil_textline_mask_tot_ea_art), 'dil_textline_mask_tot_ea_art') + tsk = time.time() + skeleton_art_textline = skeletonize(textline_mask_tot_ea_art[:,:,0]) + + skeleton_art_textline = skeleton_art_textline*1 + + skeleton_art_textline = skeleton_art_textline.astype('uint8') + + skeleton_art_textline = cv2.dilate(skeleton_art_textline, kernel, iterations=1) + + #print(np.unique(skeleton_art_textline), np.shape(skeleton_art_textline)) + + #print(skeleton_art_textline, np.unique(skeleton_art_textline)) + + #cv2.imwrite('skeleton_art_textline.png', skeleton_art_textline) + + prediction_textline[:,:,0][skeleton_art_textline[:,:]==1]=2 + + #cv2.imwrite('prediction_textline1.png', prediction_textline[:,:,0]) + + ##hor_kernel2 = cv2.getStructuringElement(cv2.MORPH_RECT, (4, 1)) + ##ver_kernel2 = cv2.getStructuringElement(cv2.MORPH_RECT, (1, 3)) + ##textline_mask_tot_ea_main = (prediction_textline[:,:]==1)*1 + ##textline_mask_tot_ea_main = textline_mask_tot_ea_main.astype('uint8') + + ##dil_textline_mask_tot_ea_main = cv2.erode(textline_mask_tot_ea_main, ver_kernel2, iterations=1) + + ##dil_textline_mask_tot_ea_main = cv2.dilate(textline_mask_tot_ea_main, hor_kernel2, iterations=1) + + ##dil_textline_mask_tot_ea_main = cv2.dilate(textline_mask_tot_ea_main, ver_kernel2, iterations=1) + + ##prediction_textline[:,:][dil_textline_mask_tot_ea_main[:,:]==1]=1 + + """ + textline_mask_tot_ea_lines = (prediction_textline[:,:]==1)*1 textline_mask_tot_ea_lines = textline_mask_tot_ea_lines.astype('uint8') if not self.textline_light: @@ -1631,10 +1852,15 @@ class Eynollah: prediction_textline[:,:][textline_mask_tot_ea_lines[:,:]==1]=1 if not self.textline_light: prediction_textline[:,:][old_art[:,:]==1]=2 + + #cv2.imwrite('prediction_textline2.png', prediction_textline[:,:,0]) prediction_textline_longshot = self.do_prediction(False, img, self.model_textline) prediction_textline_longshot_true_size = resize_image(prediction_textline_longshot, img_h, img_w) - + + + #cv2.imwrite('prediction_textline.png', prediction_textline[:,:,0]) + #sys.exit() self.logger.debug('exit textline_contours') return ((prediction_textline[:, :, 0]==1).astype(np.uint8), (prediction_textline_longshot_true_size[:, :, 0]==1).astype(np.uint8)) @@ -1840,7 +2066,7 @@ class Eynollah: textline_mask_tot_ea = resize_image(textline_mask_tot_ea,img_height_h, img_width_h ) #print(self.image_org.shape) - #cv2.imwrite('out_13.png', self.image_page_org_size) + #cv2.imwrite('textline.png', textline_mask_tot_ea) #plt.imshwo(self.image_page_org_size) #plt.show() @@ -1852,13 +2078,13 @@ class Eynollah: img_resized.shape[1], img_resized.shape[0], num_col_classifier) prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( True, img_resized, self.model_region_1_2, n_batch_inference=1, - thresholding_for_some_classes_in_light_version=True) + thresholding_for_some_classes_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) else: prediction_regions_org = np.zeros((self.image_org.shape[0], self.image_org.shape[1], 3)) confidence_matrix = np.zeros((self.image_org.shape[0], self.image_org.shape[1])) prediction_regions_page, confidence_matrix_page = self.do_prediction_new_concept( False, self.image_page_org_size, self.model_region_1_2, n_batch_inference=1, - thresholding_for_artificial_class_in_light_version=True) + thresholding_for_artificial_class_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) ys = slice(*self.page_coord[0:2]) xs = slice(*self.page_coord[2:4]) prediction_regions_org[ys, xs] = prediction_regions_page @@ -1871,7 +2097,7 @@ class Eynollah: img_resized.shape[1], img_resized.shape[0], new_h, num_col_classifier) prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( True, img_resized, self.model_region_1_2, n_batch_inference=2, - thresholding_for_some_classes_in_light_version=True) + thresholding_for_some_classes_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, n_batch_inference=3, thresholding_for_some_classes_in_light_version=True) #print("inside 3 ", time.time()-t_in) #plt.imshow(prediction_regions_org[:,:,0]) @@ -3811,7 +4037,7 @@ class Eynollah: if dilation_m1<6: dilation_m1 = 6 #print(dilation_m1, 'dilation_m1') - dilation_m1 = 6 + dilation_m1 = 4#6 dilation_m2 = int(dilation_m1/2.) +1 for i in range(len(x_differential)): @@ -4322,6 +4548,8 @@ class Eynollah: cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(textline_mask_tot_ea) all_found_textline_polygons = filter_contours_area_of_image( textline_mask_tot_ea, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001) + + all_found_textline_polygons = all_found_textline_polygons[::-1] all_found_textline_polygons=[ all_found_textline_polygons ] @@ -4329,8 +4557,8 @@ class Eynollah: all_found_textline_polygons) all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( all_found_textline_polygons, None, textline_mask_tot_ea, type_contour="textline") - - + + order_text_new = [0] slopes =[0] id_of_texts_tot =['region_0001'] @@ -4343,7 +4571,7 @@ class Eynollah: polygons_lines_xml = [] contours_tables = [] ocr_all_textlines = None - conf_contours_textregions =None + conf_contours_textregions =[0] pcgts = self.writer.build_pagexml_no_full_layout( cont_page, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, @@ -4905,7 +5133,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_ens_ocrcnn_125_225"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( From 4ddc84dee87ed7e1b600592ba8e96cad93e653e3 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 12 May 2025 18:31:40 +0200 Subject: [PATCH 129/374] visulizing textline detection from eynollah page-xml output --- train/generate_gt_for_training.py | 48 +++++++++++++++++ train/gt_gen_utils.py | 88 +++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 77e9238..9ce743a 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -2,6 +2,7 @@ import click import json from gt_gen_utils import * from tqdm import tqdm +from pathlib import Path @click.group() def main(): @@ -331,6 +332,53 @@ def visualize_reading_order(dir_xml, dir_out, dir_imgs): cv2.imwrite(os.path.join(dir_out, f_name+'.png'), img) +@main.command() +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out", + "-do", + help="directory where plots will be written", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_imgs", + "-dimg", + help="directory of images where textline segmentation will be overlayed", ) + +def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): + xml_files_ind = os.listdir(dir_xml) + for ind_xml in tqdm(xml_files_ind): + indexer = 0 + #print(ind_xml) + #print('########################') + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = Path(ind_xml).stem + + img_file_name_with_format = find_format_of_given_filename_in_dir(dir_imgs, f_name) + img = cv2.imread(os.path.join(dir_imgs, img_file_name_with_format)) + + co_tetxlines, y_len, x_len = get_textline_contours_for_visualization(xml_file) + + img_total = np.zeros((y_len, x_len, 3)) + for cont in co_tetxlines: + img_in = np.zeros((y_len, x_len, 3)) + img_in = cv2.fillPoly(img_in, pts =[cont], color=(1,1,1)) + + img_total = img_total + img_in + + img_total[:,:, 0][img_total[:,:, 0]>2] = 2 + + img_out, _ = visualize_model_output(img_total, img, task="textline") + + cv2.imwrite(os.path.join(dir_out, f_name+'.png'), img_out) + if __name__ == "__main__": main() diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 10183d6..0a65f05 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -16,6 +16,52 @@ KERNEL = np.ones((5, 5), np.uint8) with warnings.catch_warnings(): warnings.simplefilter("ignore") +def visualize_model_output(prediction, img, task): + if task == "binarization": + prediction = prediction * -1 + prediction = prediction + 1 + added_image = prediction * 255 + layout_only = None + else: + unique_classes = np.unique(prediction[:,:,0]) + rgb_colors = {'0' : [255, 255, 255], + '1' : [255, 0, 0], + '2' : [255, 125, 0], + '3' : [255, 0, 125], + '4' : [125, 125, 125], + '5' : [125, 125, 0], + '6' : [0, 125, 255], + '7' : [0, 125, 0], + '8' : [125, 125, 125], + '9' : [0, 125, 255], + '10' : [125, 0, 125], + '11' : [0, 255, 0], + '12' : [0, 0, 255], + '13' : [0, 255, 255], + '14' : [255, 125, 125], + '15' : [255, 0, 255]} + + layout_only = np.zeros(prediction.shape) + + for unq_class in unique_classes: + rgb_class_unique = rgb_colors[str(int(unq_class))] + layout_only[:,:,0][prediction[:,:,0]==unq_class] = rgb_class_unique[0] + layout_only[:,:,1][prediction[:,:,0]==unq_class] = rgb_class_unique[1] + layout_only[:,:,2][prediction[:,:,0]==unq_class] = rgb_class_unique[2] + + + + img = resize_image(img, layout_only.shape[0], layout_only.shape[1]) + + layout_only = layout_only.astype(np.int32) + img = img.astype(np.int32) + + + + added_image = cv2.addWeighted(img,0.5,layout_only,0.1,0) + + return added_image, layout_only + def get_content_of_dir(dir_in): """ Listing all ground truth page xml files. All files are needed to have xml format. @@ -138,6 +184,48 @@ def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y img_boundary[:,:][boundary[:,:]==1] =1 return co_text_eroded, img_boundary + +def get_textline_contours_for_visualization(xml_file): + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + + region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) + tag_endings = ['}TextLine','}textline'] + co_use_case = [] + + for tag in region_tags: + if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_use_case.append(np.array(c_t_in)) + return co_use_case, y_len, x_len + + def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_file, config_params, printspace, dir_images, dir_out_images): """ Reading the page xml files and write the ground truth images into given output directory. From 4a7728bb346aeccf76a34a6e0ec900e4df40a765 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 12 May 2025 22:39:47 +0200 Subject: [PATCH 130/374] visuliazation layout from eynollah page-xml output --- train/generate_gt_for_training.py | 53 ++++- train/gt_gen_utils.py | 312 ++++++++++++++++++++++++++++++ 2 files changed, 355 insertions(+), 10 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 9ce743a..7e7c6a0 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -366,18 +366,51 @@ def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): co_tetxlines, y_len, x_len = get_textline_contours_for_visualization(xml_file) - img_total = np.zeros((y_len, x_len, 3)) - for cont in co_tetxlines: - img_in = np.zeros((y_len, x_len, 3)) - img_in = cv2.fillPoly(img_in, pts =[cont], color=(1,1,1)) - - img_total = img_total + img_in - - img_total[:,:, 0][img_total[:,:, 0]>2] = 2 + added_image = visualize_image_from_contours(co_tetxlines, img) - img_out, _ = visualize_model_output(img_total, img, task="textline") + cv2.imwrite(os.path.join(dir_out, f_name+'.png'), added_image) + - cv2.imwrite(os.path.join(dir_out, f_name+'.png'), img_out) + +@main.command() +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out", + "-do", + help="directory where plots will be written", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_imgs", + "-dimg", + help="directory of images where textline segmentation will be overlayed", ) + +def visualize_layout_segmentation(dir_xml, dir_out, dir_imgs): + xml_files_ind = os.listdir(dir_xml) + for ind_xml in tqdm(xml_files_ind): + indexer = 0 + #print(ind_xml) + #print('########################') + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = Path(ind_xml).stem + + img_file_name_with_format = find_format_of_given_filename_in_dir(dir_imgs, f_name) + img = cv2.imread(os.path.join(dir_imgs, img_file_name_with_format)) + + co_text, co_graphic, co_sep, co_img, co_table, co_noise, y_len, x_len = get_layout_contours_for_visualization(xml_file) + + + added_image = visualize_image_from_contours_layout(co_text['paragraph'], co_text['header'], co_text['drop-capital'], co_sep, co_img, co_text['marginalia'], img) + + cv2.imwrite(os.path.join(dir_out, f_name+'.png'), added_image) + if __name__ == "__main__": diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 0a65f05..9b67563 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -15,6 +15,63 @@ KERNEL = np.ones((5, 5), np.uint8) with warnings.catch_warnings(): warnings.simplefilter("ignore") + + +def visualize_image_from_contours_layout(co_par, co_header, co_drop, co_sep, co_image, co_marginal, img): + alpha = 0.5 + + blank_image = np.ones( (img.shape[:]), dtype=np.uint8) * 255 + + col_header = (173, 216, 230) + col_drop = (0, 191, 255) + boundary_color = (143, 216, 200)#(0, 0, 255) # Dark gray for the boundary + col_par = (0, 0, 139) # Lighter gray for the filled area + col_image = (0, 100, 0) + col_sep = (255, 0, 0) + col_marginal = (106, 90, 205) + + if len(co_image)>0: + cv2.drawContours(blank_image, co_image, -1, col_image, thickness=cv2.FILLED) # Fill the contour + + if len(co_sep)>0: + cv2.drawContours(blank_image, co_sep, -1, col_sep, thickness=cv2.FILLED) # Fill the contour + + + if len(co_header)>0: + cv2.drawContours(blank_image, co_header, -1, col_header, thickness=cv2.FILLED) # Fill the contour + + if len(co_par)>0: + cv2.drawContours(blank_image, co_par, -1, col_par, thickness=cv2.FILLED) # Fill the contour + + cv2.drawContours(blank_image, co_par, -1, boundary_color, thickness=1) # Draw the boundary + + if len(co_drop)>0: + cv2.drawContours(blank_image, co_drop, -1, col_drop, thickness=cv2.FILLED) # Fill the contour + + if len(co_marginal)>0: + cv2.drawContours(blank_image, co_marginal, -1, col_marginal, thickness=cv2.FILLED) # Fill the contour + + img_final =cv2.cvtColor(blank_image, cv2.COLOR_BGR2RGB) + + added_image = cv2.addWeighted(img,alpha,img_final,1- alpha,0) + return added_image + + +def visualize_image_from_contours(contours, img): + alpha = 0.5 + + blank_image = np.ones( (img.shape[:]), dtype=np.uint8) * 255 + + boundary_color = (0, 0, 255) # Dark gray for the boundary + fill_color = (173, 216, 230) # Lighter gray for the filled area + + cv2.drawContours(blank_image, contours, -1, fill_color, thickness=cv2.FILLED) # Fill the contour + cv2.drawContours(blank_image, contours, -1, boundary_color, thickness=1) # Draw the boundary + + img_final =cv2.cvtColor(blank_image, cv2.COLOR_BGR2RGB) + + added_image = cv2.addWeighted(img,alpha,img_final,1- alpha,0) + return added_image def visualize_model_output(prediction, img, task): if task == "binarization": @@ -224,7 +281,262 @@ def get_textline_contours_for_visualization(xml_file): break co_use_case.append(np.array(c_t_in)) return co_use_case, y_len, x_len + + +def get_layout_contours_for_visualization(xml_file): + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + co_text = {'drop-capital':[], "footnote":[], "footnote-continued":[], "heading":[], "signature-mark":[], "header":[], "catch-word":[], "page-number":[], "marginalia":[], "paragraph":[]} + all_defined_textregion_types = list(co_text.keys()) + co_graphic = {"handwritten-annotation":[], "decoration":[], "stamp":[], "signature":[]} + all_defined_graphic_types = list(co_graphic.keys()) + co_sep=[] + co_img=[] + co_table=[] + co_noise=[] + types_text = [] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + for nn in root1.iter(tag): + c_t_in = {'drop-capital':[], "footnote":[], "footnote-continued":[], "heading":[], "signature-mark":[], "header":[], "catch-word":[], "page-number":[], "marginalia":[], "paragraph":[]} + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + + if "rest_as_paragraph" in types_text: + types_text_without_paragraph = [element for element in types_text if element!='rest_as_paragraph' and element!='paragraph'] + if len(types_text_without_paragraph) == 0: + if "type" in nn.attrib: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + elif len(types_text_without_paragraph) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_text_without_paragraph: + c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + else: + if "type" in nn.attrib: + if nn.attrib['type'] in all_defined_textregion_types: + c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + break + else: + pass + + + if vv.tag==link+'Point': + if "rest_as_paragraph" in types_text: + types_text_without_paragraph = [element for element in types_text if element!='rest_as_paragraph' and element!='paragraph'] + if len(types_text_without_paragraph) == 0: + if "type" in nn.attrib: + c_t_in['paragraph'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + elif len(types_text_without_paragraph) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_text_without_paragraph: + c_t_in[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + else: + c_t_in['paragraph'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + + else: + if "type" in nn.attrib: + if nn.attrib['type'] in all_defined_textregion_types: + c_t_in[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + + + elif vv.tag!=link+'Point' and sumi>=1: + break + + for element_text in list(c_t_in.keys()): + if len(c_t_in[element_text])>0: + co_text[element_text].append(np.array(c_t_in[element_text])) + + + if tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in_graphic = {"handwritten-annotation":[], "decoration":[], "stamp":[], "signature":[]} + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + + if "rest_as_decoration" in types_graphic: + types_graphic_without_decoration = [element for element in types_graphic if element!='rest_as_decoration' and element!='decoration'] + if len(types_graphic_without_decoration) == 0: + if "type" in nn.attrib: + c_t_in_graphic['decoration'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + elif len(types_graphic_without_decoration) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_graphic_without_decoration: + c_t_in_graphic[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in_graphic['decoration'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + else: + if "type" in nn.attrib: + if nn.attrib['type'] in all_defined_graphic_types: + c_t_in_graphic[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + break + else: + pass + + + if vv.tag==link+'Point': + if "rest_as_decoration" in types_graphic: + types_graphic_without_decoration = [element for element in types_graphic if element!='rest_as_decoration' and element!='decoration'] + if len(types_graphic_without_decoration) == 0: + if "type" in nn.attrib: + c_t_in_graphic['decoration'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + elif len(types_graphic_without_decoration) >= 1: + if "type" in nn.attrib: + if nn.attrib['type'] in types_graphic_without_decoration: + c_t_in_graphic[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + else: + c_t_in_graphic['decoration'].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + + else: + if "type" in nn.attrib: + if nn.attrib['type'] in all_defined_graphic_types: + c_t_in_graphic[nn.attrib['type']].append( [ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ] ) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + + for element_graphic in list(c_t_in_graphic.keys()): + if len(c_t_in_graphic[element_graphic])>0: + co_graphic[element_graphic].append(np.array(c_t_in_graphic[element_graphic])) + + + if tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + + + if tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + if tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + + + if tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): + #print('sth') + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + #print(vv.tag,'in') + elif vv.tag!=link+'Point' and sumi>=1: + break + co_noise.append(np.array(c_t_in)) + return co_text, co_graphic, co_sep, co_img, co_table, co_noise, y_len, x_len def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_file, config_params, printspace, dir_images, dir_out_images): """ From 54088c6b04bb64976e9873195965ade8803b7d67 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 13 May 2025 14:40:57 +0200 Subject: [PATCH 131/374] The initial attempt at reading heavily deskewed or vertically aligned lines. --- src/eynollah/eynollah.py | 91 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0c7c5d2..9f2ca50 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -22,7 +22,6 @@ from multiprocessing import cpu_count import gc import copy import json - from loky import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 @@ -77,7 +76,8 @@ from .utils.contour import ( from .utils.rotate import ( rotate_image, rotation_not_90_func, - rotation_not_90_func_full_layout + rotation_not_90_func_full_layout, + rotation_image_new ) from .utils.separate_lines import ( textline_contours_postprocessing, @@ -5310,6 +5310,75 @@ class Eynollah_ocr: img_fin = img_fin / 255. return img_fin + def get_deskewed_contour_and_bb_and_image(self, contour, image, deskew_angle): + (h_in, w_in) = image.shape[:2] + center = (w_in // 2, h_in // 2) + + rotation_matrix = cv2.getRotationMatrix2D(center, deskew_angle, 1.0) + + cos_angle = abs(rotation_matrix[0, 0]) + sin_angle = abs(rotation_matrix[0, 1]) + new_w = int((h_in * sin_angle) + (w_in * cos_angle)) + new_h = int((h_in * cos_angle) + (w_in * sin_angle)) + + rotation_matrix[0, 2] += (new_w / 2) - center[0] + rotation_matrix[1, 2] += (new_h / 2) - center[1] + + deskewed_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h)) + + contour_points = np.array(contour, dtype=np.float32) + transformed_points = cv2.transform(np.array([contour_points]), rotation_matrix)[0] + + x, y, w, h = cv2.boundingRect(np.array(transformed_points, dtype=np.int32)) + cropped_textline = deskewed_image[y:y+h, x:x+w] + + return cropped_textline + + def rotate_image_with_padding(self, image, angle): + # Get image dimensions + (h, w) = image.shape[:2] + + # Calculate the center of the image + center = (w // 2, h // 2) + + # Get the rotation matrix + rotation_matrix = cv2.getRotationMatrix2D(center, angle, 1.0) + + # Compute the new bounding dimensions + cos = abs(rotation_matrix[0, 0]) + sin = abs(rotation_matrix[0, 1]) + new_w = int((h * sin) + (w * cos)) + new_h = int((h * cos) + (w * sin)) + + # Adjust the rotation matrix to account for translation + rotation_matrix[0, 2] += (new_w / 2) - center[0] + rotation_matrix[1, 2] += (new_h / 2) - center[1] + + # Perform the rotation + rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=(0, 0, 0)) + + return rotated_image + + def get_orientation_moments(self, contour): + moments = cv2.moments(contour) + if moments["mu20"] - moments["mu02"] == 0: # Avoid division by zero + return 90 if moments["mu11"] > 0 else -90 + else: + angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) + return np.degrees(angle) # Convert radians to degrees + + def get_contours_and_bounding_boxes(self, mask): + # Find contours in the binary mask + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + largest_contour = max(contours, key=cv2.contourArea) if contours else None + + # Get the bounding rectangle for the contour + x, y, w, h = cv2.boundingRect(largest_contour) + #bounding_boxes.append((x, y, w, h)) + + return x, y, w, h + def run(self): ls_imgs = os.listdir(self.dir_in) @@ -5533,6 +5602,10 @@ class Eynollah_ocr: x,y,w,h = cv2.boundingRect(textline_coords) + angle_radians = math.atan2(h, w) + # Convert to degrees + angle_degrees = math.degrees(angle_radians) + if self.draw_texts_on_image: total_bb_coordinates.append([x,y,w,h]) @@ -5549,7 +5622,21 @@ class Eynollah_ocr: mask_poly = mask_poly[y:y+h, x:x+w, :] img_crop = img_poly_on_img[y:y+h, x:x+w, :] if not self.do_not_mask_with_textline_contour: + if angle_degrees > 15: + better_des_slope = self.get_orientation_moments(textline_coords) + + img_crop = self.rotate_image_with_padding(img_crop, -abs(better_des_slope) ) + mask_poly = self.rotate_image_with_padding(mask_poly, -abs(better_des_slope) ) + mask_poly = mask_poly.astype('uint8') + + #new bounding box + x_n, y_n, w_n, h_n = self.get_contours_and_bounding_boxes(mask_poly[:,:,0]) + + mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_crop[mask_poly==0] = 255 + if self.prediction_with_both_of_rgb_and_bin: img_crop_bin[mask_poly==0] = 255 From 88e03153217eba0809c53d617387d3cf3403a7c2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 13 May 2025 15:53:05 +0200 Subject: [PATCH 132/374] Accurately writing text line contours into xml file when the deskewing exceeds 45 degrees and the text line is in light mode --- src/eynollah/writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 92e353f..8cd1c8e 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -119,7 +119,7 @@ class EynollahXmlWriter(): points_co += ',' points_co += str(textline_y_coord) - if (self.curved_line or self.textline_light) and np.abs(slopes[region_idx]) <= 45: + if self.textline_light or (self.curved_line and np.abs(slopes[region_idx]) <= 45): if len(contour_textline) == 2: points_co += str(int((contour_textline[0] + page_coord[2]) / self.scale_x)) points_co += ',' @@ -128,7 +128,7 @@ class EynollahXmlWriter(): points_co += str(int((contour_textline[0][0] + page_coord[2]) / self.scale_x)) points_co += ',' points_co += str(int((contour_textline[0][1] + page_coord[0])/self.scale_y)) - elif (self.curved_line or self.textline_light) and np.abs(slopes[region_idx]) > 45: + elif self.curved_line and np.abs(slopes[region_idx]) > 45: if len(contour_textline)==2: points_co += str(int((contour_textline[0] + region_bboxes[2] + page_coord[2])/self.scale_x)) points_co += ',' From 25abc0fabc8a70b6a9c21c35006c08fec577d792 Mon Sep 17 00:00:00 2001 From: johnlockejrr <16368414+johnlockejrr@users.noreply.github.com> Date: Wed, 14 May 2025 03:34:51 -0700 Subject: [PATCH 133/374] Update gt_gen_utils.py Keep safely the full basename without extension --- train/gt_gen_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 5784e14..8837462 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -22,7 +22,7 @@ def get_content_of_dir(dir_in): """ gt_all=os.listdir(dir_in) - gt_list=[file for file in gt_all if file.split('.')[ len(file.split('.'))-1 ]=='xml' ] + gt_list = [file for file in gt_all if os.path.splitext(file)[1] == '.xml'] return gt_list def return_parent_contours(contours, hierarchy): @@ -134,7 +134,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ if dir_images: ls_org_imgs = os.listdir(dir_images) - ls_org_imgs_stem = [item.split('.')[0] for item in ls_org_imgs] + ls_org_imgs_stem = [os.path.splitext(item)[0] for item in ls_org_imgs] for index in tqdm(range(len(gt_list))): #try: tree1 = ET.parse(dir_in+'/'+gt_list[index], parser = ET.XMLParser(encoding = 'iso-8859-5')) @@ -298,10 +298,10 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly = resize_image(img_poly, y_new, x_new) try: - xml_file_stem = gt_list[index].split('-')[1].split('.')[0] + xml_file_stem = os.path.splitext(gt_list[index])[0] cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) except: - xml_file_stem = gt_list[index].split('.')[0] + xml_file_stem = os.path.splitext(gt_list[index])[0] cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) if dir_images: @@ -757,10 +757,10 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ img_poly = resize_image(img_poly, y_new, x_new) try: - xml_file_stem = gt_list[index].split('-')[1].split('.')[0] + xml_file_stem = os.path.splitext(gt_list[index])[0] cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) except: - xml_file_stem = gt_list[index].split('.')[0] + xml_file_stem = os.path.splitext(gt_list[index])[0] cv2.imwrite(os.path.join(output_dir, xml_file_stem + '.png'), img_poly) From ed46615f004a96191208d2f5481229003336644f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 14 May 2025 18:34:58 +0200 Subject: [PATCH 134/374] enhance ocr for vertical textlines --- src/eynollah/eynollah.py | 79 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9f2ca50..5a73ef3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5133,7 +5133,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_ens_ocrcnn_125_225"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_425000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5585,6 +5585,7 @@ class Eynollah_ocr: region_tags=np.unique([x for x in alltags if x.endswith('TextRegion')]) cropped_lines = [] + cropped_lines_ver_index = [] cropped_lines_region_indexer = [] cropped_lines_meging_indexing = [] @@ -5644,6 +5645,11 @@ class Eynollah_ocr: if w_scaled < 1.5*image_width: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) + if angle_degrees > 15: + cropped_lines_ver_index.append(1) + else: + cropped_lines_ver_index.append(0) + cropped_lines_meging_indexing.append(0) if self.prediction_with_both_of_rgb_and_bin: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) @@ -5657,11 +5663,22 @@ class Eynollah_ocr: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(1) + + if angle_degrees > 15: + cropped_lines_ver_index.append(1) + else: + cropped_lines_ver_index.append(0) + img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(-1) + if angle_degrees > 15: + cropped_lines_ver_index.append(1) + else: + cropped_lines_ver_index.append(0) + if self.prediction_with_both_of_rgb_and_bin: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[0], image_height, image_width) cropped_lines_bin.append(img_fin) @@ -5673,6 +5690,11 @@ class Eynollah_ocr: cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) + if angle_degrees > 15: + cropped_lines_ver_index.append(1) + else: + cropped_lines_ver_index.append(0) + if self.prediction_with_both_of_rgb_and_bin: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) @@ -5722,6 +5744,19 @@ class Eynollah_ocr: imgs = cropped_lines[n_start:] imgs = np.array(imgs) imgs = imgs.reshape(imgs.shape[0], image_height, image_width, 3) + + ver_imgs = np.array( cropped_lines_ver_index[n_start:] ) + indices_ver = np.where(ver_imgs == 1)[0] + + #print(indices_ver, 'indices_ver') + if len(indices_ver)>0: + imgs_ver_flipped = imgs[indices_ver, : ,: ,:] + imgs_ver_flipped = imgs_ver_flipped[:,::-1,::-1,:] + #print(imgs_ver_flipped, 'imgs_ver_flipped') + + else: + imgs_ver_flipped = None + if self.prediction_with_both_of_rgb_and_bin: imgs_bin = cropped_lines_bin[n_start:] imgs_bin = np.array(imgs_bin) @@ -5732,12 +5767,54 @@ class Eynollah_ocr: imgs = cropped_lines[n_start:n_end] imgs = np.array(imgs).reshape(self.b_s, image_height, image_width, 3) + ver_imgs = np.array( cropped_lines_ver_index[n_start:n_end] ) + indices_ver = np.where(ver_imgs == 1)[0] + #print(indices_ver, 'indices_ver') + + if len(indices_ver)>0: + imgs_ver_flipped = imgs[indices_ver, : ,: ,:] + imgs_ver_flipped = imgs_ver_flipped[:,::-1,::-1,:] + #print(imgs_ver_flipped, 'imgs_ver_flipped') + else: + imgs_ver_flipped = None + + if self.prediction_with_both_of_rgb_and_bin: imgs_bin = cropped_lines_bin[n_start:n_end] imgs_bin = np.array(imgs_bin).reshape(self.b_s, image_height, image_width, 3) preds = self.prediction_model.predict(imgs, verbose=0) + + if len(indices_ver)>0: + #cv2.imwrite('flipped.png', (imgs_ver_flipped[0, :,:,:]*255).astype('uint8')) + #cv2.imwrite('original.png', (imgs[0, :,:,:]*255).astype('uint8')) + #sys.exit() + #print(imgs_ver_flipped.shape, 'imgs_ver_flipped.shape') + preds_flipped = self.prediction_model.predict(imgs_ver_flipped, verbose=0) + preds_max_fliped = np.max(preds_flipped, axis=2 ) + preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) + pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=256 + masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) + masked_means_flipped[np.isnan(masked_means_flipped)] = 0 + #print(masked_means_flipped, 'masked_means_flipped') + + preds_max = np.max(preds, axis=2 ) + preds_max_args = np.argmax(preds, axis=2 ) + pred_max_not_unk_mask_bool = preds_max_args[:,:]!=256 + + masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) + masked_means[np.isnan(masked_means)] = 0 + + masked_means_ver = masked_means[indices_ver] + #print(masked_means_ver, 'pred_max_not_unk') + + indices_where_flipped_conf_value_is_higher = np.where(masked_means_flipped > masked_means_ver)[0] + + #print(indices_where_flipped_conf_value_is_higher, 'indices_where_flipped_conf_value_is_higher') + if len(indices_where_flipped_conf_value_is_higher)>0: + indices_to_be_replaced = indices_ver[indices_where_flipped_conf_value_is_higher] + preds[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] if self.prediction_with_both_of_rgb_and_bin: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) preds = (preds + preds_bin) / 2. From 7a22e51f5d2ebff1bd0239c913eb1ed13d97fe77 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 14 May 2025 21:56:03 +0200 Subject: [PATCH 135/374] resolve some comments from review --- README.md | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3cfb587..8a2c4a4 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,8 @@ * Output in [PAGE-XML](https://github.com/PRImA-Research-Lab/PAGE-XML) * [OCR-D](https://github.com/qurator-spk/eynollah#use-as-ocr-d-processor) interface -:warning: Development is currently focused on achieving the best possible quality of results for a wide variety of historical documents and therefore processing can be very slow. We aim to improve this, but contributions are welcome. +:warning: Development is currently focused on achieving the best possible quality of results for a wide variety of +historical documents and therefore processing can be very slow. We aim to improve this, but contributions are welcome. ## Installation Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported. @@ -42,7 +43,7 @@ cd eynollah; pip install -e . Alternatively, you can run `make install` or `make install-dev` for editable installation. ## Models -Pre-trained models can be downloaded from [qurator-data.de](https://qurator-data.de/eynollah/) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). +Pretrained models can be downloaded from [qurator-data.de](https://qurator-data.de/eynollah/) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). @@ -50,13 +51,17 @@ For documentation on methods and models, have a look at [`models.md`](https://gi In case you want to train your own model with Eynollah, have a look at [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md). ## Usage +Eynollah supports four use cases: layout analysis (segmentation), binarization, text recognition (OCR), +and (trainable) reading order detection. -Eynollah has four key use cases: layout analysis, binarization, OCR, and machine-based reading order. +### Layout Analysis +The layout analysis module is responsible for detecting layouts, identifying text lines, and determining reading order +using both heuristic methods or a machine-based reading order detection model. -### Layout -The layout module is responsible for detecting layouts, identifying text lines, and determining reading order using both heuristic methods or a machine-based reading order detection model. It's important to note that this functionality should not be confused with the machine-based-reading-order use case. The latter, still under development, focuses specifically on determining the reading order for a given layout in an XML file. In contrast, layout detection takes an image as input, and after detecting the layout, it can also determine the reading order using a machine-based model. +Note that there are currently two supported ways for reading order detection: either as part of layout analysis based +on image input, or, currently under development, for given layout analysis results based on PAGE-XML data as input. -The command-line interface for layout can be called like this: +The command-line interface for layout analysis can be called like this: ```sh eynollah layout \ @@ -87,18 +92,19 @@ The following options can be used to further configure the processing: | `-sp ` | save cropped page image to this directory | | `-sa ` | save all (plot, enhanced/binary image, layout) to this directory | -If no option is set, the tool performs layout detection of main regions (background, text, images, separators and marginals). +If no option is set, the tool performs layout detection of main regions (background, text, images, separators +and marginals). The best output quality is produced when RGB images are used as input rather than greyscale or binarized images. ### Binarization -Document Image Binarization +The binarization module performs document image binarization using pretrained pixelwise segmentation models. The command-line interface for binarization of single image can be called like this: ```sh eynollah binarization \ - -m \ - \ + -m \ + \ ``` @@ -117,9 +123,7 @@ Under development ### Machine-based-reading-order Under development - #### Use as OCR-D processor - Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) [processor](https://ocr-d.de/en/spec/cli), formally described in [`ocrd-tool.json`](https://github.com/qurator-spk/eynollah/tree/main/src/eynollah/ocrd-tool.json). @@ -127,7 +131,6 @@ In this case, the source image file group with (preferably) RGB images should be ocrd-eynollah-segment -I OCR-D-IMG -O OCR-D-SEG -P models 2022-04-05 - If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynollah behaves as follows: - existing regions are kept and ignored (i.e. in effect they might overlap segments from Eynollah results) - existing annotation (and respective `AlternativeImage`s) are partially _ignored_: @@ -138,7 +141,6 @@ If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynol (because some other preprocessing step was in effect like `denoised`), then the output PAGE-XML will be based on that as new top-level (`@imageFilename`) - ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models 2022-04-05 Still, in general, it makes more sense to add other workflow steps **after** Eynollah. From 1b229ba7aeab5b5811ab3a3f0bf85ac3164ba0ec Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 15 May 2025 00:45:22 +0200 Subject: [PATCH 136/374] enhancement for vertical textlines --- src/eynollah/eynollah.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 5a73ef3..2e54687 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5626,8 +5626,8 @@ class Eynollah_ocr: if angle_degrees > 15: better_des_slope = self.get_orientation_moments(textline_coords) - img_crop = self.rotate_image_with_padding(img_crop, -abs(better_des_slope) ) - mask_poly = self.rotate_image_with_padding(mask_poly, -abs(better_des_slope) ) + img_crop = self.rotate_image_with_padding(img_crop, better_des_slope ) + mask_poly = self.rotate_image_with_padding(mask_poly, better_des_slope ) mask_poly = mask_poly.astype('uint8') #new bounding box From 1cbc669d36334d421dd2af9801e17456a35b0f01 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 15 May 2025 15:33:50 +0200 Subject: [PATCH 137/374] marginals detection enhanced for light version --- src/eynollah/utils/marginals.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/eynollah/utils/marginals.py b/src/eynollah/utils/marginals.py index a29e50d..c0c4892 100644 --- a/src/eynollah/utils/marginals.py +++ b/src/eynollah/utils/marginals.py @@ -26,8 +26,10 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve text_with_lines=resize_image(text_with_lines,int(text_with_lines.shape[0]*1.8),text_with_lines.shape[1]) text_with_lines=cv2.erode(text_with_lines,kernel,iterations=7) text_with_lines=resize_image(text_with_lines,text_with_lines_eroded.shape[0],text_with_lines_eroded.shape[1]) - - + + if light_version: + text_with_lines=rotate_image(text_with_lines,-slope_deskew) + text_with_lines_y=text_with_lines.sum(axis=0) text_with_lines_y_eroded=text_with_lines_eroded.sum(axis=0) From f9390c71e7ec3c577e80ad4a8894417481407f02 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sat, 17 May 2025 02:18:27 +0200 Subject: [PATCH 138/374] updating inference for mb reading order --- train/gt_gen_utils.py | 2 +- train/inference.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 9b67563..a734020 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -154,7 +154,7 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m jv += 1 return found_polygons_early -def filter_contours_area_of_image(image, contours, order_index, max_area, min_area, min_early): +def filter_contours_area_of_image(image, contours, order_index, max_area, min_area, min_early=None): found_polygons_early = list() order_index_filtered = list() regions_ar_less_than_early_min = list() diff --git a/train/inference.py b/train/inference.py index db3b31f..aecd0e6 100644 --- a/train/inference.py +++ b/train/inference.py @@ -267,7 +267,7 @@ class sbb_predict: #print(np.shape(co_text_all[0]), len( np.shape(co_text_all[0]) ),'co_text_all') #co_text_all = filter_contours_area_of_image_tables(img_poly, co_text_all, _, max_area, min_area) #print(co_text_all,'co_text_all') - co_text_all, texts_corr_order_index_int = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, self.min_area) + co_text_all, texts_corr_order_index_int, _ = filter_contours_area_of_image(img_poly, co_text_all, texts_corr_order_index_int, max_area, self.min_area) #print(texts_corr_order_index_int) From 5016039cd74098b30ccc78b9f0d0bdf0bb91f351 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 18 May 2025 02:48:05 +0200 Subject: [PATCH 139/374] enhancing marginal detection for light version --- src/eynollah/eynollah.py | 7 +++---- src/eynollah/utils/marginals.py | 13 ++++++++----- src/eynollah/utils/separate_lines.py | 4 ++-- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 2e54687..08a781c 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -272,7 +272,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_step_2500000_mb_ro"#"/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" @@ -1315,7 +1315,7 @@ class Eynollah: seg_art[seg_art>0] =1 seg_line = label_p_pred[:,:,:,3] - seg_line[seg_line>0.5] =1#seg_line[seg_line>0.1] =1 + seg_line[seg_line>0.3] =1#seg_line[seg_line>0.5] =1#seg_line[seg_line>0.1] =1 seg_line[seg_line<1] =0 ##seg[seg_art==1]=4 @@ -3667,7 +3667,6 @@ class Eynollah: peaks_real, _ = find_peaks(sum_smoothed, height=0) if len(peaks_real)>70: - print(len(peaks_real), 'len(peaks_real)') peaks_real = peaks_real[(peaks_realwidth1)] @@ -5133,7 +5132,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_425000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_600000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( diff --git a/src/eynollah/utils/marginals.py b/src/eynollah/utils/marginals.py index c0c4892..ac8dc1d 100644 --- a/src/eynollah/utils/marginals.py +++ b/src/eynollah/utils/marginals.py @@ -10,7 +10,6 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve mask_marginals=np.zeros((text_with_lines.shape[0],text_with_lines.shape[1])) mask_marginals=mask_marginals.astype(np.uint8) - text_with_lines=text_with_lines.astype(np.uint8) ##text_with_lines=cv2.erode(text_with_lines,self.kernel,iterations=3) @@ -26,9 +25,11 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve text_with_lines=resize_image(text_with_lines,int(text_with_lines.shape[0]*1.8),text_with_lines.shape[1]) text_with_lines=cv2.erode(text_with_lines,kernel,iterations=7) text_with_lines=resize_image(text_with_lines,text_with_lines_eroded.shape[0],text_with_lines_eroded.shape[1]) - + + if light_version: - text_with_lines=rotate_image(text_with_lines,-slope_deskew) + kernel_hor = np.ones((1, 5), dtype=np.uint8) + text_with_lines = cv2.erode(text_with_lines,kernel_hor,iterations=6) text_with_lines_y=text_with_lines.sum(axis=0) text_with_lines_y_eroded=text_with_lines_eroded.sum(axis=0) @@ -42,8 +43,10 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve elif thickness_along_y_percent>=30 and thickness_along_y_percent<50: min_textline_thickness=20 else: - min_textline_thickness=40 - + if light_version: + min_textline_thickness=45 + else: + min_textline_thickness=40 if thickness_along_y_percent>=14: diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 0322579..6289d4d 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1466,7 +1466,7 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, main_page=False, logger=None, plotter=None, map=map): if main_page and plotter: plotter.save_plot_of_textline_density(img_patch_org) - + img_int=np.zeros((img_patch_org.shape[0],img_patch_org.shape[1])) img_int[:,:]=img_patch_org[:,:]#img_patch_org[:,:,0] @@ -1487,7 +1487,7 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles) angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) elif main_page: - angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) + angles = np.array (list(np.linspace(-12, -7, int(n_tot_angles/4))) + list(np.linspace(-6, 6, n_tot_angles- 2* int(n_tot_angles/4))) + list(np.linspace(7, 12, int(n_tot_angles/4))))#np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) early_slope_edge=11 From 44ff51f5c17fb1836f76b3ea953e7470521d3300 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 20 May 2025 16:51:08 +0200 Subject: [PATCH 140/374] mb reading order now can be done faster. Text regions are clustered using dilation, and mb reading order needs to be implemented for fewer regions --- src/eynollah/eynollah.py | 181 +++++++++++++++++++++++++++++---- src/eynollah/utils/__init__.py | 2 +- 2 files changed, 163 insertions(+), 20 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 08a781c..eb5c860 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -32,6 +32,7 @@ from numba import cuda from skimage.morphology import skeletonize from ocrd import OcrdPage from ocrd_utils import getLogger, tf_disable_interactive_logs +import statistics try: import torch @@ -797,7 +798,7 @@ class Eynollah: self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False, threshold_art_class_textline=0.1): + thresholding_for_artificial_class_in_light_version=False, thresholding_for_fl_light_version=False, threshold_art_class_textline=0.1): self.logger.debug("enter do_prediction") img_height_model = model.layers[-1].output_shape[1] @@ -822,6 +823,15 @@ class Eynollah: skeleton_art = skeleton_art*1 seg[skeleton_art==1]=2 + + if thresholding_for_fl_light_version: + seg_header = label_p_pred[0,:,:,2] + + seg_header[seg_header<0.2] = 0 + seg_header[seg_header>0] =1 + + seg[seg_header==1]=2 + seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) return prediction_true @@ -1613,10 +1623,11 @@ class Eynollah: model_region = self.model_region_fl if patches else self.model_region_fl_np if self.light_version: - pass + thresholding_for_fl_light_version = True elif not patches: img = otsu_copy_binary(img).astype(np.uint8) prediction_regions = None + thresholding_for_fl_light_version = False elif cols: img = otsu_copy_binary(img).astype(np.uint8) if cols == 1: @@ -1632,7 +1643,7 @@ class Eynollah: else: img = resize_image(img, int(img_height_h * 2500 / float(img_width_h)), 2500).astype(np.uint8) - prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1, n_batch_inference=3) + prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1, n_batch_inference=3, thresholding_for_fl_light_version=thresholding_for_fl_light_version) prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) self.logger.debug("exit extract_text_regions") return prediction_regions, prediction_regions @@ -3544,9 +3555,87 @@ class Eynollah: return model def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p): + #cv2.imwrite('textregions.png', text_regions_p*50) + min_cont_size_to_be_dilated = 10 + if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + ver_kernel = np.ones((5, 1), dtype=np.uint8) + + cx_conts, cy_conts, x_min_conts, x_max_conts, y_min_conts, y_max_conts, _ = find_new_features_of_contours(contours_only_text_parent) + args_cont_located = np.array(range(len(contours_only_text_parent))) + + diff_y_conts = np.abs(y_max_conts[:]-y_min_conts) + diff_x_conts = np.abs(x_max_conts[:]-x_min_conts) + + mean_x = statistics.mean(diff_x_conts) + median_x = statistics.median(diff_x_conts) + + + diff_x_ratio= diff_x_conts/mean_x + + args_cont_located_excluded = args_cont_located[diff_x_ratio>=1.3] + args_cont_located_included = args_cont_located[diff_x_ratio<1.3] + + contours_only_text_parent_excluded = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]>=1.3]#contours_only_text_parent[diff_x_ratio>=1.3] + contours_only_text_parent_included = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]<1.3]#contours_only_text_parent[diff_x_ratio<1.3] + + + cx_conts_excluded = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]>=1.3]#cx_conts[diff_x_ratio>=1.3] + cx_conts_included = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]<1.3]#cx_conts[diff_x_ratio<1.3] + + cy_conts_excluded = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]>=1.3]#cy_conts[diff_x_ratio>=1.3] + cy_conts_included = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]<1.3]#cy_conts[diff_x_ratio<1.3] + + #print(diff_x_ratio, 'ratio') + text_regions_p = text_regions_p.astype('uint8') + + if len(contours_only_text_parent_excluded)>0: + textregion_par = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1])).astype('uint8') + textregion_par = cv2.fillPoly(textregion_par, pts=contours_only_text_parent_included, color=(1,1)) + else: + textregion_par = (text_regions_p[:,:]==1)*1 + textregion_par = textregion_par.astype('uint8') + + + text_regions_p_textregions_dilated = cv2.dilate(textregion_par , ver_kernel, iterations=8) + text_regions_p_textregions_dilated[text_regions_p[:,:]>1] = 0 + + #cv2.imwrite('textregions_dilated.png', text_regions_p_textregions_dilated*255) + + + contours_only_dilated, hir_on_text_dilated = return_contours_of_image(text_regions_p_textregions_dilated) + contours_only_dilated = return_parent_contours(contours_only_dilated, hir_on_text_dilated) + + indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located = self.return_indexes_of_contours_loctaed_inside_another_list_of_contours(contours_only_dilated, contours_only_text_parent_included, cx_conts_included, cy_conts_included, args_cont_located_included) + + + if len(args_cont_located_excluded)>0: + for ind in args_cont_located_excluded: + indexes_of_located_cont.append(np.array([ind])) + contours_only_dilated.append(contours_only_text_parent[ind]) + center_y_coordinates_of_located.append(0) + + array_list = [np.array([elem]) if isinstance(elem, int) else elem for elem in indexes_of_located_cont] + flattened_array = np.concatenate([arr.ravel() for arr in array_list]) + #print(len( np.unique(flattened_array)), 'indexes_of_located_cont uniques') + + missing_textregions = list( set(np.array(range(len(contours_only_text_parent))) ) - set(np.unique(flattened_array)) ) + #print(missing_textregions, 'missing_textregions') + + for ind in missing_textregions: + indexes_of_located_cont.append(np.array([ind])) + contours_only_dilated.append(contours_only_text_parent[ind]) + center_y_coordinates_of_located.append(0) + + + if contours_only_text_parent_h: + for vi in range(len(contours_only_text_parent_h)): + indexes_of_located_cont.append(int(vi+len(contours_only_text_parent))) + + array_list = [np.array([elem]) if isinstance(elem, int) else elem for elem in indexes_of_located_cont] + flattened_array = np.concatenate([arr.ravel() for arr in array_list]) + y_len = text_regions_p.shape[0] x_len = text_regions_p.shape[1] - img_poly = np.zeros((y_len,x_len), dtype='uint8') img_poly[text_regions_p[:,:]==1] = 1 @@ -3554,25 +3643,24 @@ class Eynollah: img_poly[text_regions_p[:,:]==3] = 4 img_poly[text_regions_p[:,:]==6] = 5 - - ###temp - ##sep_mask = (img_poly==5)*1 - ##sep_mask = sep_mask.astype('uint8') - ##sep_mask = cv2.erode(sep_mask, kernel=KERNEL, iterations=2) - ##img_poly[img_poly==5] = 0 - ##img_poly[sep_mask==1] = 5 - ### - img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') if contours_only_text_parent_h: _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours( contours_only_text_parent_h) for j in range(len(cy_main)): img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12, - int(x_min_main[j]):int(x_max_main[j])] = 1 - co_text_all = contours_only_text_parent + contours_only_text_parent_h + int(x_min_main[j]):int(x_max_main[j])] = 1 + co_text_all_org = contours_only_text_parent + contours_only_text_parent_h + if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + co_text_all = contours_only_dilated + contours_only_text_parent_h + else: + co_text_all = contours_only_text_parent + contours_only_text_parent_h else: - co_text_all = contours_only_text_parent + co_text_all_org = contours_only_text_parent + if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + co_text_all = contours_only_dilated + else: + co_text_all = contours_only_text_parent if not len(co_text_all): return [], [] @@ -3651,8 +3739,26 @@ class Eynollah: break ordered = [i[0] for i in ordered] - region_ids = ['region_%04d' % i for i in range(len(co_text_all))] - return ordered, region_ids + + if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + org_contours_indexes = [] + for ind in range(len(ordered)): + region_with_curr_order = ordered[ind] + if region_with_curr_order < len(contours_only_dilated): + if np.isscalar(indexes_of_located_cont[region_with_curr_order]): + org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]] + else: + arg_sort_located_cont = np.argsort(center_y_coordinates_of_located[region_with_curr_order]) + org_contours_indexes = org_contours_indexes + list(np.array(indexes_of_located_cont[region_with_curr_order])[arg_sort_located_cont]) ##org_contours_indexes + list ( + else: + org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]] + + region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] + return org_contours_indexes, region_ids + else: + region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] + return ordered, region_ids + def return_start_and_end_of_common_text_of_textline_ocr(self, textline_image, ind_tot): width = np.shape(textline_image)[1] @@ -4293,6 +4399,29 @@ class Eynollah: contours[ind_u_a_trs].pop(ittrd) return contours + + def return_indexes_of_contours_loctaed_inside_another_list_of_contours(self, contours, contours_loc, cx_main_loc, cy_main_loc, indexes_loc): + indexes_of_located_cont = [] + center_x_coordinates_of_located = [] + center_y_coordinates_of_located = [] + #M_main_tot = [cv2.moments(contours_loc[j]) + #for j in range(len(contours_loc))] + #cx_main_loc = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] + #cy_main_loc = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] + + for ij in range(len(contours)): + results = [cv2.pointPolygonTest(contours[ij], (cx_main_loc[ind], cy_main_loc[ind]), False) + for ind in range(len(cy_main_loc)) ] + results = np.array(results) + indexes_in = np.where((results == 0) | (results == 1)) + indexes = indexes_loc[indexes_in]# [(results == 0) | (results == 1)]#np.where((results == 0) | (results == 1)) + + indexes_of_located_cont.append(indexes) + center_x_coordinates_of_located.append(np.array(cx_main_loc)[indexes_in] ) + center_y_coordinates_of_located.append(np.array(cy_main_loc)[indexes_in] ) + + return indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located + def filter_contours_without_textline_inside( self, contours,text_con_org, contours_textline, contours_only_text_parent_d_ordered, conf_contours_textregions): @@ -4986,8 +5115,10 @@ class Eynollah: if self.full_layout: if self.reading_order_machine_based: + tror = time.time() order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( contours_only_text_parent, contours_only_text_parent_h, text_regions_p) + print('time spend for mb ro', time.time()-tror) else: if np.abs(slope_deskew) < SLOPE_THRESHOLD: order_text_new, id_of_texts_tot = self.do_order_of_regions( @@ -5619,8 +5750,15 @@ class Eynollah_ocr: mask_poly = np.zeros(img.shape) mask_poly = cv2.fillPoly(mask_poly, pts=[textline_coords], color=(1, 1, 1)) + mask_poly = mask_poly[y:y+h, x:x+w, :] img_crop = img_poly_on_img[y:y+h, x:x+w, :] + + if angle_degrees<=15: + if mask_poly[:,:,0].sum() /float(w*h) < 0.6 and w_scaled > 520: + cv2.imwrite(file_name+'_desk.png', img_crop) + + print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') if not self.do_not_mask_with_textline_contour: if angle_degrees > 15: better_des_slope = self.get_orientation_moments(textline_coords) @@ -5634,6 +5772,11 @@ class Eynollah_ocr: mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] + + if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.6 and w_scaled > 520: + cv2.imwrite(file_name+'_desk.png', img_crop) + + print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii') img_crop[mask_poly==0] = 255 @@ -5641,7 +5784,7 @@ class Eynollah_ocr: img_crop_bin[mask_poly==0] = 255 if not self.export_textline_images_and_text: - if w_scaled < 1.5*image_width: + if w_scaled < 640:#1.5*image_width: img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) if angle_degrees > 15: diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index c5962f8..7fa4a7b 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -992,7 +992,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light( (regions_model_full[:,:,0]==2)).sum() pixels_main = all_pixels - pixels_header - if (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): + if ( (pixels_header/float(pixels_main)>=0.6) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ) and ( (length_con[ii]/float(height_con[ii]) )<=3 )) or ( (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=3 ) ): regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 contours_only_text_parent_head.append(con) if contours_only_text_parent_d_ordered is not None: From 3ad621e956dd7cdb8e7f2d00edcfa4db7008d7d9 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 20 May 2025 19:01:52 +0200 Subject: [PATCH 141/374] ocr for curved lines --- src/eynollah/eynollah.py | 157 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 146 insertions(+), 11 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index eb5c860..912bc31 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5263,7 +5263,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_600000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_750000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5464,7 +5464,7 @@ class Eynollah_ocr: return cropped_textline - def rotate_image_with_padding(self, image, angle): + def rotate_image_with_padding(self, image, angle, border_value=(0,0,0)): # Get image dimensions (h, w) = image.shape[:2] @@ -5485,7 +5485,7 @@ class Eynollah_ocr: rotation_matrix[1, 2] += (new_h / 2) - center[1] # Perform the rotation - rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=(0, 0, 0)) + rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=border_value) return rotated_image @@ -5496,6 +5496,21 @@ class Eynollah_ocr: else: angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) return np.degrees(angle) # Convert radians to degrees + + + def get_orientation_moments_of_mask(self, mask): + mask=mask.astype('uint8') + print(mask.shape) + contours, _ = cv2.findContours(mask[:,:,0], cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + largest_contour = max(contours, key=cv2.contourArea) if contours else None + + moments = cv2.moments(largest_contour) + if moments["mu20"] - moments["mu02"] == 0: # Avoid division by zero + return 90 if moments["mu11"] > 0 else -90 + else: + angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) + return np.degrees(angle) # Convert radians to degrees def get_contours_and_bounding_boxes(self, mask): # Find contours in the binary mask @@ -5508,6 +5523,121 @@ class Eynollah_ocr: #bounding_boxes.append((x, y, w, h)) return x, y, w, h + + def return_splitting_point_of_image(self, image_to_spliited): + width = np.shape(image_to_spliited)[1] + height = np.shape(image_to_spliited)[0] + common_window = int(0.03*width) + + width1 = int ( common_window) + width2 = int ( width - common_window ) + + img_sum = np.sum(image_to_spliited[:,:,0], axis=0) + sum_smoothed = gaussian_filter1d(img_sum, 3) + + peaks_real, _ = find_peaks(sum_smoothed, height=0) + + peaks_real = peaks_real[(peaks_realwidth1)] + + arg_sort = np.argsort(sum_smoothed[peaks_real]) + arg_sort4 =arg_sort[::-1][:4] + peaks_sort_4 = peaks_real[arg_sort][::-1][:4] + + return np.sort(peaks_sort_4) + + def break_curved_line_into_small_pieces_and_then_merge(self, img_curved, mask_curved): + peaks_4 = self.return_splitting_point_of_image(img_curved) + + + + img_0 = img_curved[:, :peaks_4[0], :] + img_1 = img_curved[:, peaks_4[0]:peaks_4[1], :] + img_2 = img_curved[:, peaks_4[1]:peaks_4[2], :] + img_3 = img_curved[:, peaks_4[2]:peaks_4[3], :] + img_4 = img_curved[:, peaks_4[3]:, :] + + + mask_0 = mask_curved[:, :peaks_4[0], :] + mask_1 = mask_curved[:, peaks_4[0]:peaks_4[1], :] + mask_2 = mask_curved[:, peaks_4[1]:peaks_4[2], :] + mask_3 = mask_curved[:, peaks_4[2]:peaks_4[3], :] + mask_4 = mask_curved[:, peaks_4[3]:, :] + + cv2.imwrite("split0.png", img_0) + cv2.imwrite("split1.png", img_1) + cv2.imwrite("split2.png", img_2) + cv2.imwrite("split3.png", img_3) + + or_ma_0 = self.get_orientation_moments_of_mask(mask_0) + or_ma_1 = self.get_orientation_moments_of_mask(mask_1) + or_ma_2 = self.get_orientation_moments_of_mask(mask_2) + or_ma_3 = self.get_orientation_moments_of_mask(mask_3) + or_ma_4 = self.get_orientation_moments_of_mask(mask_4) + + imgs_tot = [] + imgs_tot.append([img_0, mask_0, or_ma_0] ) + imgs_tot.append([img_1, mask_1, or_ma_1]) + imgs_tot.append([img_2, mask_2, or_ma_2]) + imgs_tot.append([img_3, mask_3, or_ma_3]) + imgs_tot.append([img_4, mask_4, or_ma_4]) + + w_tot_des_list = [] + w_tot_des = 0 + imgs_deskewed_list = [] + for ind in range(len(imgs_tot)): + img_in = imgs_tot[ind][0] + mask_in = imgs_tot[ind][1] + ori_in = imgs_tot[ind][2] + + if abs(ori_in)<45: + img_in_des = self.rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) + mask_in_des = self.rotate_image_with_padding(mask_in, ori_in) + mask_in_des = mask_in_des.astype('uint8') + + #new bounding box + x_n, y_n, w_n, h_n = self.get_contours_and_bounding_boxes(mask_in_des[:,:,0]) + + mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + img_in_des = resize_image(img_in_des, 32, w_relative) + + + else: + img_in_des = np.copy(img_in) + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + img_in_des = resize_image(img_in_des, 32, w_relative) + + w_tot_des+=img_in_des.shape[1] + w_tot_des_list.append(img_in_des.shape[1]) + imgs_deskewed_list.append(img_in_des) + + + + + img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 + + w_indexer = 0 + for ind in range(len(w_tot_des_list)): + img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] + w_indexer = w_indexer+w_tot_des_list[ind] + + #cv2.imwrite('final.png', img_final_deskewed) + #print(or_ma_0, or_ma_1, or_ma_2, or_ma_3, or_ma_4, 'orients') + + ##cv2.imwrite("split4.png", img_curved[:, peaks_4[3]:peaks_4[4], :]) + ##cv2.imwrite("split5.png", img_curved[:, peaks_4[4]:peaks_4[5], :]) + ##cv2.imwrite("split6.png", img_curved[:, peaks_4[5]:peaks_4[6], :]) + + ##cv2.imwrite("split7.png", img_curved[:, peaks_4[6]:peaks_4[7], :]) + ##cv2.imwrite("split8.png", img_curved[:, peaks_4[7]:peaks_4[8], :]) + ##cv2.imwrite("split9.png", img_curved[:, peaks_4[8]:peaks_4[9], :]) + + + #cv2.imwrite("split4.png", img_4) + #sys.exit() + return img_final_deskewed def run(self): ls_imgs = os.listdir(self.dir_in) @@ -5754,11 +5884,9 @@ class Eynollah_ocr: mask_poly = mask_poly[y:y+h, x:x+w, :] img_crop = img_poly_on_img[y:y+h, x:x+w, :] - if angle_degrees<=15: - if mask_poly[:,:,0].sum() /float(w*h) < 0.6 and w_scaled > 520: - cv2.imwrite(file_name+'_desk.png', img_crop) + - print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') + #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') if not self.do_not_mask_with_textline_contour: if angle_degrees > 15: better_des_slope = self.get_orientation_moments(textline_coords) @@ -5773,12 +5901,19 @@ class Eynollah_ocr: mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] - if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.6 and w_scaled > 520: - cv2.imwrite(file_name+'_desk.png', img_crop) + img_crop[mask_poly==0] = 255 - print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii') + if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 100: + img_crop = self.break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) - img_crop[mask_poly==0] = 255 + #print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii') + else: + img_crop[mask_poly==0] = 255 + if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 100: + img_crop = self.break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + + + if self.prediction_with_both_of_rgb_and_bin: img_crop_bin[mask_poly==0] = 255 From 14b70c25565d595ad31f6b1ce2e77df491f78679 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 21 May 2025 14:39:31 +0200 Subject: [PATCH 142/374] Implement hyphenated textline merging in OCR engine and a bug fixed for curved textline OCR --- src/eynollah/eynollah.py | 157 ++++++++++++++++++--------------------- 1 file changed, 71 insertions(+), 86 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 912bc31..6771db0 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5500,7 +5500,6 @@ class Eynollah_ocr: def get_orientation_moments_of_mask(self, mask): mask=mask.astype('uint8') - print(mask.shape) contours, _ = cv2.findContours(mask[:,:,0], cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) largest_contour = max(contours, key=cv2.contourArea) if contours else None @@ -5547,97 +5546,69 @@ class Eynollah_ocr: def break_curved_line_into_small_pieces_and_then_merge(self, img_curved, mask_curved): peaks_4 = self.return_splitting_point_of_image(img_curved) - - - - img_0 = img_curved[:, :peaks_4[0], :] - img_1 = img_curved[:, peaks_4[0]:peaks_4[1], :] - img_2 = img_curved[:, peaks_4[1]:peaks_4[2], :] - img_3 = img_curved[:, peaks_4[2]:peaks_4[3], :] - img_4 = img_curved[:, peaks_4[3]:, :] - - - mask_0 = mask_curved[:, :peaks_4[0], :] - mask_1 = mask_curved[:, peaks_4[0]:peaks_4[1], :] - mask_2 = mask_curved[:, peaks_4[1]:peaks_4[2], :] - mask_3 = mask_curved[:, peaks_4[2]:peaks_4[3], :] - mask_4 = mask_curved[:, peaks_4[3]:, :] - - cv2.imwrite("split0.png", img_0) - cv2.imwrite("split1.png", img_1) - cv2.imwrite("split2.png", img_2) - cv2.imwrite("split3.png", img_3) - - or_ma_0 = self.get_orientation_moments_of_mask(mask_0) - or_ma_1 = self.get_orientation_moments_of_mask(mask_1) - or_ma_2 = self.get_orientation_moments_of_mask(mask_2) - or_ma_3 = self.get_orientation_moments_of_mask(mask_3) - or_ma_4 = self.get_orientation_moments_of_mask(mask_4) - - imgs_tot = [] - imgs_tot.append([img_0, mask_0, or_ma_0] ) - imgs_tot.append([img_1, mask_1, or_ma_1]) - imgs_tot.append([img_2, mask_2, or_ma_2]) - imgs_tot.append([img_3, mask_3, or_ma_3]) - imgs_tot.append([img_4, mask_4, or_ma_4]) - - w_tot_des_list = [] - w_tot_des = 0 - imgs_deskewed_list = [] - for ind in range(len(imgs_tot)): - img_in = imgs_tot[ind][0] - mask_in = imgs_tot[ind][1] - ori_in = imgs_tot[ind][2] + if len(peaks_4)>0: + imgs_tot = [] - if abs(ori_in)<45: - img_in_des = self.rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) - mask_in_des = self.rotate_image_with_padding(mask_in, ori_in) - mask_in_des = mask_in_des.astype('uint8') + for ind in range(len(peaks_4)+1): + if ind==0: + img = img_curved[:, :peaks_4[ind], :] + mask = mask_curved[:, :peaks_4[ind], :] + elif ind==len(peaks_4): + img = img_curved[:, peaks_4[ind-1]:, :] + mask = mask_curved[:, peaks_4[ind-1]:, :] + else: + img = img_curved[:, peaks_4[ind-1]:peaks_4[ind], :] + mask = mask_curved[:, peaks_4[ind-1]:peaks_4[ind], :] + + or_ma = self.get_orientation_moments_of_mask(mask) + + imgs_tot.append([img, mask, or_ma] ) + + + w_tot_des_list = [] + w_tot_des = 0 + imgs_deskewed_list = [] + for ind in range(len(imgs_tot)): + img_in = imgs_tot[ind][0] + mask_in = imgs_tot[ind][1] + ori_in = imgs_tot[ind][2] - #new bounding box - x_n, y_n, w_n, h_n = self.get_contours_and_bounding_boxes(mask_in_des[:,:,0]) + if abs(ori_in)<45: + img_in_des = self.rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) + mask_in_des = self.rotate_image_with_padding(mask_in, ori_in) + mask_in_des = mask_in_des.astype('uint8') + + #new bounding box + x_n, y_n, w_n, h_n = self.get_contours_and_bounding_boxes(mask_in_des[:,:,0]) + + mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + img_in_des = resize_image(img_in_des, 32, w_relative) + + + else: + img_in_des = np.copy(img_in) + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + img_in_des = resize_image(img_in_des, 32, w_relative) + + w_tot_des+=img_in_des.shape[1] + w_tot_des_list.append(img_in_des.shape[1]) + imgs_deskewed_list.append(img_in_des) - mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) - img_in_des = resize_image(img_in_des, 32, w_relative) - else: - img_in_des = np.copy(img_in) - w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) - img_in_des = resize_image(img_in_des, 32, w_relative) - - w_tot_des+=img_in_des.shape[1] - w_tot_des_list.append(img_in_des.shape[1]) - imgs_deskewed_list.append(img_in_des) + img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 - - - - img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 - - w_indexer = 0 - for ind in range(len(w_tot_des_list)): - img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] - w_indexer = w_indexer+w_tot_des_list[ind] - - #cv2.imwrite('final.png', img_final_deskewed) - #print(or_ma_0, or_ma_1, or_ma_2, or_ma_3, or_ma_4, 'orients') - - ##cv2.imwrite("split4.png", img_curved[:, peaks_4[3]:peaks_4[4], :]) - ##cv2.imwrite("split5.png", img_curved[:, peaks_4[4]:peaks_4[5], :]) - ##cv2.imwrite("split6.png", img_curved[:, peaks_4[5]:peaks_4[6], :]) - - ##cv2.imwrite("split7.png", img_curved[:, peaks_4[6]:peaks_4[7], :]) - ##cv2.imwrite("split8.png", img_curved[:, peaks_4[7]:peaks_4[8], :]) - ##cv2.imwrite("split9.png", img_curved[:, peaks_4[8]:peaks_4[9], :]) - - - #cv2.imwrite("split4.png", img_4) - #sys.exit() - return img_final_deskewed + w_indexer = 0 + for ind in range(len(w_tot_des_list)): + img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] + w_indexer = w_indexer+w_tot_des_list[ind] + return img_final_deskewed + else: + return img_curved def run(self): ls_imgs = os.listdir(self.dir_in) @@ -6144,7 +6115,21 @@ class Eynollah_ocr: text_by_textregion = [] for ind in unique_cropped_lines_region_indexer: extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] - text_by_textregion.append("".join(extracted_texts_merged_un)) + if len(extracted_texts_merged_un)>1: + text_by_textregion_ind = "" + next_glue = "" + for indt in range(len(extracted_texts_merged_un)): + if extracted_texts_merged_un[indt].endswith('⸗') or extracted_texts_merged_un[indt].endswith('-'): + text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt][:-1] + next_glue = "" + else: + text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt] + next_glue = " " + text_by_textregion.append(text_by_textregion_ind) + + else: + text_by_textregion.append(" ".join(extracted_texts_merged_un)) + #print(text_by_textregion, 'text_by_textregiontext_by_textregiontext_by_textregiontext_by_textregiontext_by_textregion') indexer = 0 indexer_textregion = 0 From ee2c7e90137988e83fe3f2204c8a46849cce0f19 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 21 May 2025 17:42:44 +0200 Subject: [PATCH 143/374] enhancing curved lines OCR --- src/eynollah/eynollah.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6771db0..b510218 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5532,14 +5532,12 @@ class Eynollah_ocr: width2 = int ( width - common_window ) img_sum = np.sum(image_to_spliited[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) + sum_smoothed = gaussian_filter1d(img_sum, 1) peaks_real, _ = find_peaks(sum_smoothed, height=0) - peaks_real = peaks_real[(peaks_realwidth1)] - + arg_sort = np.argsort(sum_smoothed[peaks_real]) - arg_sort4 =arg_sort[::-1][:4] peaks_sort_4 = peaks_real[arg_sort][::-1][:4] return np.sort(peaks_sort_4) @@ -5585,12 +5583,16 @@ class Eynollah_ocr: img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + if w_relative==0: + w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) else: img_in_des = np.copy(img_in) w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + if w_relative==0: + w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) w_tot_des+=img_in_des.shape[1] From 089029cec734b254ba9737b0d213f49d1d16beef Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 23 May 2025 15:55:03 +0200 Subject: [PATCH 144/374] commit 21ec4fb is picked + rnn ocr at the same time with segmentation + enhancement of mb reading order --- src/eynollah/cli.py | 15 +- src/eynollah/eynollah.py | 775 +++++++++++--------------------- src/eynollah/utils/utils_ocr.py | 435 ++++++++++++++++++ src/eynollah/writer.py | 30 +- 4 files changed, 729 insertions(+), 526 deletions(-) create mode 100644 src/eynollah/utils/utils_ocr.py diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 99961c9..cd56833 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -225,6 +225,17 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) is_flag=True, help="if this parameter set to true, this tool will try to do ocr", ) +@click.option( + "--transformer_ocr", + "-tr/-notr", + is_flag=True, + help="if this parameter set to true, this tool will apply transformer ocr", +) +@click.option( + "--batch_size_ocr", + "-bs_ocr", + help="number of inference batch size of ocr model. Default b_s for trocr and cnn_rnn models are 2 and 8 respectively", +) @click.option( "--num_col_upper", "-ncu", @@ -258,7 +269,7 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) help="Override log level globally to this", ) -def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, num_col_upper, num_col_lower, threshold_art_class_textline, threshold_art_class_layout, skip_layout_and_reading_order, ignore_page_extraction, log_level): +def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, transformer_ocr, batch_size_ocr, num_col_upper, num_col_lower, threshold_art_class_textline, threshold_art_class_layout, skip_layout_and_reading_order, ignore_page_extraction, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) @@ -305,6 +316,8 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ ignore_page_extraction=ignore_page_extraction, reading_order_machine_based=reading_order_machine_based, do_ocr=do_ocr, + transformer_ocr=transformer_ocr, + batch_size_ocr=batch_size_ocr, num_col_upper=num_col_upper, num_col_lower=num_col_lower, skip_layout_and_reading_order=skip_layout_and_reading_order, diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index b510218..2564150 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -80,6 +80,13 @@ from .utils.rotate import ( rotation_not_90_func_full_layout, rotation_image_new ) +from .utils.utils_ocr import ( + return_textline_contour_with_added_box_coordinate, + preprocess_and_resize_image_for_ocrcnn_model, + return_textlines_split_if_needed, + decode_batch_predictions, + return_rnn_cnn_ocr_of_given_textlines +) from .utils.separate_lines import ( textline_contours_postprocessing, separate_lines_new2, @@ -199,6 +206,8 @@ class Eynollah: ignore_page_extraction : bool = False, reading_order_machine_based : bool = False, do_ocr : bool = False, + transformer_ocr: bool = False, + batch_size_ocr: Optional[int] = None, num_col_upper : Optional[int] = None, num_col_lower : Optional[int] = None, threshold_art_class_layout: Optional[float] = None, @@ -232,6 +241,7 @@ class Eynollah: self.ignore_page_extraction = ignore_page_extraction self.skip_layout_and_reading_order = skip_layout_and_reading_order self.ocr = do_ocr + self.tr = transformer_ocr if num_col_upper: self.num_col_upper = int(num_col_upper) else: @@ -273,7 +283,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_step_2500000_mb_ro"#"/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_step_4800000_mb_ro"#"/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" @@ -300,8 +310,10 @@ class Eynollah: else: #"/eynollah-textline_20210425" self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024" - if self.ocr: + if self.ocr and self.tr: self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" + elif self.ocr and not self.tr: + self.model_ocr_dir = dir_models + "/model_step_750000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" if self.tables: if self.light_version: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" @@ -341,11 +353,37 @@ class Eynollah: self.model_region_fl = self.our_load_model(self.model_region_dir_fully) if self.reading_order_machine_based: self.model_reading_order = self.our_load_model(self.model_reading_order_dir) - if self.ocr: + if self.ocr and self.tr: self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") #("microsoft/trocr-base-printed")#("microsoft/trocr-base-handwritten") self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") + elif self.ocr and not self.tr: + model_ocr = load_model(self.model_ocr_dir , compile=False) + + self.prediction_model = tf.keras.models.Model( + model_ocr.get_layer(name = "image").input, + model_ocr.get_layer(name = "dense2").output) + if not batch_size_ocr: + self.b_s_ocr = 8 + else: + self.b_s_ocr = int(batch_size_ocr) + + + with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: + characters = json.load(config_file) + + + AUTOTUNE = tf.data.AUTOTUNE + + # Mapping characters to integers. + char_to_num = StringLookup(vocabulary=list(characters), mask_token=None) + + # Mapping integers back to original characters. + self.num_to_char = StringLookup( + vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True + ) + if self.tables: self.model_table = self.our_load_model(self.model_table_dir) @@ -1325,11 +1363,11 @@ class Eynollah: seg_art[seg_art>0] =1 seg_line = label_p_pred[:,:,:,3] - seg_line[seg_line>0.3] =1#seg_line[seg_line>0.5] =1#seg_line[seg_line>0.1] =1 + seg_line[seg_line>0.4] =1#seg_line[seg_line>0.5] =1#seg_line[seg_line>0.1] =1 seg_line[seg_line<1] =0 ##seg[seg_art==1]=4 - seg[(seg_line==1) & (seg==0)]=3 + #seg[(seg_line==1) & (seg==0)]=3 if thresholding_for_artificial_class_in_light_version: seg_art = label_p_pred[:,:,:,2] @@ -2060,7 +2098,7 @@ class Eynollah: ###img_bin = np.copy(prediction_bin) ###else: ###img_bin = np.copy(img_resized) - if self.ocr and not self.input_binary: + if (self.ocr and self.tr) and not self.input_binary: prediction_bin = self.do_prediction(True, img_resized, self.model_bin, n_batch_inference=5) prediction_bin = 255 * (prediction_bin[:,:,0] == 0) prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) @@ -3485,8 +3523,10 @@ class Eynollah: # 6 is the separators lable in old full layout model # 4 is the drop capital class in old full layout model # in the new full layout drop capital is 3 and separators are 5 - - text_regions_p[:,:][regions_fully[:,:,0]==5]=6 + + # the separators in full layout will not be written on layout + if not self.reading_order_machine_based: + text_regions_p[:,:][regions_fully[:,:,0]==5]=6 ###regions_fully[:, :, 0][regions_fully_only_drop[:, :, 0] == 3] = 4 #text_regions_p[:,:][regions_fully[:,:,0]==6]=6 @@ -3555,11 +3595,37 @@ class Eynollah: return model def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p): - #cv2.imwrite('textregions.png', text_regions_p*50) + + height1 =672#448 + width1 = 448#224 + + height2 =672#448 + width2= 448#224 + + height3 =672#448 + width3 = 448#224 + + inference_bs = 3 + + cv2.imwrite('textregions.png', text_regions_p*50) + cv2.imwrite('sep.png', (text_regions_p[:,:]==6)*255) + + ver_kernel = np.ones((5, 1), dtype=np.uint8) + hor_kernel = np.ones((1, 5), dtype=np.uint8) + + + + #separators = (text_regions_p[:,:]==6)*1 + #text_regions_p[text_regions_p[:,:]==6] = 0 + #separators = separators.astype('uint8') + + #separators = cv2.erode(separators , hor_kernel, iterations=1) + #text_regions_p[separators[:,:]==1] = 6 + + #cv2.imwrite('sep_new.png', (text_regions_p[:,:]==6)*255) + min_cont_size_to_be_dilated = 10 if len(contours_only_text_parent)>min_cont_size_to_be_dilated: - ver_kernel = np.ones((5, 1), dtype=np.uint8) - cx_conts, cy_conts, x_min_conts, x_max_conts, y_min_conts, y_max_conts, _ = find_new_features_of_contours(contours_only_text_parent) args_cont_located = np.array(range(len(contours_only_text_parent))) @@ -3595,12 +3661,13 @@ class Eynollah: textregion_par = (text_regions_p[:,:]==1)*1 textregion_par = textregion_par.astype('uint8') - - text_regions_p_textregions_dilated = cv2.dilate(textregion_par , ver_kernel, iterations=8) + text_regions_p_textregions_dilated = cv2.erode(textregion_par , hor_kernel, iterations=2) + text_regions_p_textregions_dilated = cv2.dilate(text_regions_p_textregions_dilated , ver_kernel, iterations=4) + text_regions_p_textregions_dilated = cv2.erode(text_regions_p_textregions_dilated , hor_kernel, iterations=1) + text_regions_p_textregions_dilated = cv2.dilate(text_regions_p_textregions_dilated , ver_kernel, iterations=5) text_regions_p_textregions_dilated[text_regions_p[:,:]>1] = 0 - #cv2.imwrite('textregions_dilated.png', text_regions_p_textregions_dilated*255) - + cv2.imwrite('text_regions_p_textregions_dilated.png', text_regions_p_textregions_dilated*255) contours_only_dilated, hir_on_text_dilated = return_contours_of_image(text_regions_p_textregions_dilated) contours_only_dilated = return_parent_contours(contours_only_dilated, hir_on_text_dilated) @@ -3664,7 +3731,8 @@ class Eynollah: if not len(co_text_all): return [], [] - + print(len(co_text_all), "co_text_all") + print(len(co_text_all_org), "co_text_all_org") labels_con = np.zeros((int(y_len /6.), int(x_len/6.), len(co_text_all)), dtype=bool) co_text_all = [(i/6).astype(int) for i in co_text_all] for i in range(len(co_text_all)): @@ -3675,21 +3743,13 @@ class Eynollah: cv2.fillPoly(img, pts=[co_text_all[i]], color=(1,)) labels_con[:,:,i] = img - height1 =672#448 - width1 = 448#224 - - height2 =672#448 - width2= 448#224 - - height3 =672#448 - width3 = 448#224 labels_con = resize_image(labels_con.astype(np.uint8), height1, width1).astype(bool) img_header_and_sep = resize_image(img_header_and_sep, height1, width1) img_poly = resize_image(img_poly, height3, width3) - inference_bs = 3 + input_1 = np.zeros((inference_bs, height1, width1, 3)) ordered = [list(range(len(co_text_all)))] index_update = 0 @@ -3760,217 +3820,213 @@ class Eynollah: return ordered, region_ids - def return_start_and_end_of_common_text_of_textline_ocr(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.2*width) + ####def return_start_and_end_of_common_text_of_textline_ocr(self, textline_image, ind_tot): + ####width = np.shape(textline_image)[1] + ####height = np.shape(textline_image)[0] + ####common_window = int(0.2*width) - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) + ####width1 = int ( width/2. - common_window ) + ####width2 = int ( width/2. + common_window ) - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) + ####img_sum = np.sum(textline_image[:,:,0], axis=0) + ####sum_smoothed = gaussian_filter1d(img_sum, 3) - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: + ####peaks_real, _ = find_peaks(sum_smoothed, height=0) + ####if len(peaks_real)>70: - peaks_real = peaks_real[(peaks_realwidth1)] + ####peaks_real = peaks_real[(peaks_realwidth1)] - arg_sort = np.argsort(sum_smoothed[peaks_real]) - arg_sort4 =arg_sort[::-1][:4] - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - argsort_sorted = np.argsort(peaks_sort_4) + ####arg_sort = np.argsort(sum_smoothed[peaks_real]) + ####arg_sort4 =arg_sort[::-1][:4] + ####peaks_sort_4 = peaks_real[arg_sort][::-1][:4] + ####argsort_sorted = np.argsort(peaks_sort_4) - first_4_sorted = peaks_sort_4[argsort_sorted] - y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] - #print(first_4_sorted,'first_4_sorted') + ####first_4_sorted = peaks_sort_4[argsort_sorted] + ####y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] + #####print(first_4_sorted,'first_4_sorted') - arg_sortnew = np.argsort(y_4_sorted) - peaks_final =np.sort( first_4_sorted[arg_sortnew][2:] ) + ####arg_sortnew = np.argsort(y_4_sorted) + ####peaks_final =np.sort( first_4_sorted[arg_sortnew][2:] ) - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peaks_final[0], peaks_final[0]], [0, height-1]) - #plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') + #####plt.figure(ind_tot) + #####plt.imshow(textline_image) + #####plt.plot([peaks_final[0], peaks_final[0]], [0, height-1]) + #####plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) + #####plt.savefig('./'+str(ind_tot)+'.png') - return peaks_final[0], peaks_final[1] - else: - pass + ####return peaks_final[0], peaks_final[1] + ####else: + ####pass - def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.06*width) + ##def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image, ind_tot): + ##width = np.shape(textline_image)[1] + ##height = np.shape(textline_image)[0] + ##common_window = int(0.06*width) - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) + ##width1 = int ( width/2. - common_window ) + ##width2 = int ( width/2. + common_window ) - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) + ##img_sum = np.sum(textline_image[:,:,0], axis=0) + ##sum_smoothed = gaussian_filter1d(img_sum, 3) - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: - #print(len(peaks_real), 'len(peaks_real)') + ##peaks_real, _ = find_peaks(sum_smoothed, height=0) + ##if len(peaks_real)>70: + ###print(len(peaks_real), 'len(peaks_real)') - peaks_real = peaks_real[(peaks_realwidth1)] + ##peaks_real = peaks_real[(peaks_realwidth1)] - arg_max = np.argmax(sum_smoothed[peaks_real]) - peaks_final = peaks_real[arg_max] + ##arg_max = np.argmax(sum_smoothed[peaks_real]) + ##peaks_final = peaks_real[arg_max] - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peaks_final, peaks_final], [0, height-1]) - ##plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') + ###plt.figure(ind_tot) + ###plt.imshow(textline_image) + ###plt.plot([peaks_final, peaks_final], [0, height-1]) + ####plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) + ###plt.savefig('./'+str(ind_tot)+'.png') - return peaks_final - else: - return None + ##return peaks_final + ##else: + ##return None - def return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - self, peaks_real, sum_smoothed, start_split, end_split): + ###def return_start_and_end_of_common_text_of_textline_ocr_new_splitted( + ###self, peaks_real, sum_smoothed, start_split, end_split): - peaks_real = peaks_real[(peaks_realstart_split)] + ###peaks_real = peaks_real[(peaks_realstart_split)] - arg_sort = np.argsort(sum_smoothed[peaks_real]) - arg_sort4 =arg_sort[::-1][:4] - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - argsort_sorted = np.argsort(peaks_sort_4) + ###arg_sort = np.argsort(sum_smoothed[peaks_real]) + ###arg_sort4 =arg_sort[::-1][:4] + ###peaks_sort_4 = peaks_real[arg_sort][::-1][:4] + ###argsort_sorted = np.argsort(peaks_sort_4) - first_4_sorted = peaks_sort_4[argsort_sorted] - y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] - #print(first_4_sorted,'first_4_sorted') + ###first_4_sorted = peaks_sort_4[argsort_sorted] + ###y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] + ####print(first_4_sorted,'first_4_sorted') - arg_sortnew = np.argsort(y_4_sorted) - peaks_final =np.sort( first_4_sorted[arg_sortnew][3:] ) - return peaks_final[0] + ###arg_sortnew = np.argsort(y_4_sorted) + ###peaks_final =np.sort( first_4_sorted[arg_sortnew][3:] ) + ###return peaks_final[0] - def return_start_and_end_of_common_text_of_textline_ocr_new(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.15*width) + ###def return_start_and_end_of_common_text_of_textline_ocr_new(self, textline_image, ind_tot): + ###width = np.shape(textline_image)[1] + ###height = np.shape(textline_image)[0] + ###common_window = int(0.15*width) - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - mid = int(width/2.) + ###width1 = int ( width/2. - common_window ) + ###width2 = int ( width/2. + common_window ) + ###mid = int(width/2.) - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) + ###img_sum = np.sum(textline_image[:,:,0], axis=0) + ###sum_smoothed = gaussian_filter1d(img_sum, 3) - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: - peak_start = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - peaks_real, sum_smoothed, width1, mid+2) - peak_end = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - peaks_real, sum_smoothed, mid-2, width2) + ###peaks_real, _ = find_peaks(sum_smoothed, height=0) + ###if len(peaks_real)>70: + ###peak_start = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( + ###peaks_real, sum_smoothed, width1, mid+2) + ###peak_end = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( + ###peaks_real, sum_smoothed, mid-2, width2) - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peak_start, peak_start], [0, height-1]) - #plt.plot([peak_end, peak_end], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') + ####plt.figure(ind_tot) + ####plt.imshow(textline_image) + ####plt.plot([peak_start, peak_start], [0, height-1]) + ####plt.plot([peak_end, peak_end], [0, height-1]) + ####plt.savefig('./'+str(ind_tot)+'.png') - return peak_start, peak_end - else: - pass + ###return peak_start, peak_end + ###else: + ###pass - def return_ocr_of_textline_without_common_section( - self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): + ##def return_ocr_of_textline_without_common_section( + ##self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): - if h2w_ratio > 0.05: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - else: - #width = np.shape(textline_image)[1] - #height = np.shape(textline_image)[0] - #common_window = int(0.3*width) - #width1 = int ( width/2. - common_window ) - #width2 = int ( width/2. + common_window ) + ##if h2w_ratio > 0.05: + ##pixel_values = processor(textline_image, return_tensors="pt").pixel_values + ##generated_ids = model_ocr.generate(pixel_values.to(device)) + ##generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + ##else: + ###width = np.shape(textline_image)[1] + ###height = np.shape(textline_image)[0] + ###common_window = int(0.3*width) + ###width1 = int ( width/2. - common_window ) + ###width2 = int ( width/2. + common_window ) - split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section( - textline_image, ind_tot) - if split_point: - image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) - image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) + ##split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section( + ##textline_image, ind_tot) + ##if split_point: + ##image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) + ##image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) - #pixel_values1 = processor(image1, return_tensors="pt").pixel_values - #pixel_values2 = processor(image2, return_tensors="pt").pixel_values + ###pixel_values1 = processor(image1, return_tensors="pt").pixel_values + ###pixel_values2 = processor(image2, return_tensors="pt").pixel_values - pixel_values_merged = processor([image1,image2], return_tensors="pt").pixel_values - generated_ids_merged = model_ocr.generate(pixel_values_merged.to(device)) - generated_text_merged = processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + ##pixel_values_merged = processor([image1,image2], return_tensors="pt").pixel_values + ##generated_ids_merged = model_ocr.generate(pixel_values_merged.to(device)) + ##generated_text_merged = processor.batch_decode(generated_ids_merged, skip_special_tokens=True) - #print(generated_text_merged,'generated_text_merged') + ###print(generated_text_merged,'generated_text_merged') - #generated_ids1 = model_ocr.generate(pixel_values1.to(device)) - #generated_ids2 = model_ocr.generate(pixel_values2.to(device)) + ###generated_ids1 = model_ocr.generate(pixel_values1.to(device)) + ###generated_ids2 = model_ocr.generate(pixel_values2.to(device)) - #generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] - #generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] + ###generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] + ###generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] - #generated_text = generated_text1 + ' ' + generated_text2 - generated_text = generated_text_merged[0] + ' ' + generated_text_merged[1] + ###generated_text = generated_text1 + ' ' + generated_text2 + ##generated_text = generated_text_merged[0] + ' ' + generated_text_merged[1] - #print(generated_text1,'generated_text1') - #print(generated_text2, 'generated_text2') - #print('########################################') - else: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + ###print(generated_text1,'generated_text1') + ###print(generated_text2, 'generated_text2') + ###print('########################################') + ##else: + ##pixel_values = processor(textline_image, return_tensors="pt").pixel_values + ##generated_ids = model_ocr.generate(pixel_values.to(device)) + ##generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - #print(generated_text,'generated_text') - #print('########################################') - return generated_text + ###print(generated_text,'generated_text') + ###print('########################################') + ##return generated_text - def return_ocr_of_textline( - self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): + ###def return_ocr_of_textline( + ###self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): - if h2w_ratio > 0.05: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - else: - #width = np.shape(textline_image)[1] - #height = np.shape(textline_image)[0] - #common_window = int(0.3*width) - #width1 = int ( width/2. - common_window ) - #width2 = int ( width/2. + common_window ) + ###if h2w_ratio > 0.05: + ###pixel_values = processor(textline_image, return_tensors="pt").pixel_values + ###generated_ids = model_ocr.generate(pixel_values.to(device)) + ###generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + ###else: + ####width = np.shape(textline_image)[1] + ####height = np.shape(textline_image)[0] + ####common_window = int(0.3*width) + ####width1 = int ( width/2. - common_window ) + ####width2 = int ( width/2. + common_window ) - try: - width1, width2 = self.return_start_and_end_of_common_text_of_textline_ocr_new(textline_image, ind_tot) + ###try: + ###width1, width2 = self.return_start_and_end_of_common_text_of_textline_ocr_new(textline_image, ind_tot) - image1 = textline_image[:, :width2,:]# image.crop((0, 0, width2, height)) - image2 = textline_image[:, width1:,:]#image.crop((width1, 0, width, height)) + ###image1 = textline_image[:, :width2,:]# image.crop((0, 0, width2, height)) + ###image2 = textline_image[:, width1:,:]#image.crop((width1, 0, width, height)) - pixel_values1 = processor(image1, return_tensors="pt").pixel_values - pixel_values2 = processor(image2, return_tensors="pt").pixel_values + ###pixel_values1 = processor(image1, return_tensors="pt").pixel_values + ###pixel_values2 = processor(image2, return_tensors="pt").pixel_values - generated_ids1 = model_ocr.generate(pixel_values1.to(device)) - generated_ids2 = model_ocr.generate(pixel_values2.to(device)) + ###generated_ids1 = model_ocr.generate(pixel_values1.to(device)) + ###generated_ids2 = model_ocr.generate(pixel_values2.to(device)) - generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] - generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] - #print(generated_text1,'generated_text1') - #print(generated_text2, 'generated_text2') - #print('########################################') + ###generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] + ###generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] + ####print(generated_text1,'generated_text1') + ####print(generated_text2, 'generated_text2') + ####print('########################################') - match = sq(None, generated_text1, generated_text2).find_longest_match( - 0, len(generated_text1), 0, len(generated_text2)) - generated_text = generated_text1 + generated_text2[match.b+match.size:] - except: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] + ###match = sq(None, generated_text1, generated_text2).find_longest_match( + ###0, len(generated_text1), 0, len(generated_text2)) + ###generated_text = generated_text1 + generated_text2[match.b+match.size:] + ###except: + ###pixel_values = processor(textline_image, return_tensors="pt").pixel_values + ###generated_ids = model_ocr.generate(pixel_values.to(device)) + ###generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - return generated_text + ###return generated_text - def return_textline_contour_with_added_box_coordinate(self, textline_contour, box_ind): - textline_contour[:,0] = textline_contour[:,0] + box_ind[2] - textline_contour[:,1] = textline_contour[:,1] + box_ind[0] - return textline_contour def return_list_of_contours_with_desired_order(self, ls_cons, sorted_indexes): return [ls_cons[sorted_indexes[index]] for index in range(len(sorted_indexes))] @@ -4625,6 +4681,7 @@ class Eynollah: raise ValueError("run requires either a single image filename or a directory") for img_filename in self.ls_imgs: + print(img_filename, 'img_filename') self.logger.info(img_filename) t0 = time.time() @@ -4698,13 +4755,19 @@ class Eynollah: all_box_coord_marginals = [] polygons_lines_xml = [] contours_tables = [] - ocr_all_textlines = None conf_contours_textregions =[0] + + if self.ocr and not self.tr: + gc.collect() + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, textline_light=True) + else: + ocr_all_textlines = None + pcgts = self.writer.build_pagexml_no_full_layout( cont_page, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions, self.skip_layout_and_reading_order) return pcgts #print("text region early -1 in %.1fs", time.time() - t0) @@ -5118,7 +5181,7 @@ class Eynollah: tror = time.time() order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( contours_only_text_parent, contours_only_text_parent_h, text_regions_p) - print('time spend for mb ro', time.time()-tror) + print('time spend for mb ro', time.time()-tror) else: if np.abs(slope_deskew) < SLOPE_THRESHOLD: order_text_new, id_of_texts_tot = self.do_order_of_regions( @@ -5160,7 +5223,7 @@ class Eynollah: order_text_new, id_of_texts_tot = self.do_order_of_regions( contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) - if self.ocr: + if self.ocr and self.tr: device = cuda.get_current_device() device.reset() gc.collect() @@ -5207,6 +5270,11 @@ class Eynollah: ocr_textline_in_textregion.append(text_ocr) ind_tot = ind_tot +1 ocr_all_textlines.append(ocr_textline_in_textregion) + + elif self.ocr and not self.tr: + gc.collect() + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + else: ocr_all_textlines = None @@ -5289,329 +5357,6 @@ class Eynollah_ocr: vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True ) - - def decode_batch_predictions(self, pred, max_len = 128): - # input_len is the product of the batch size and the - # number of time steps. - input_len = np.ones(pred.shape[0]) * pred.shape[1] - - # Decode CTC predictions using greedy search. - # decoded is a tuple with 2 elements. - decoded = tf.keras.backend.ctc_decode(pred, - input_length = input_len, - beam_width = 100) - # The outputs are in the first element of the tuple. - # Additionally, the first element is actually a list, - # therefore we take the first element of that list as well. - #print(decoded,'decoded') - decoded = decoded[0][0][:, :max_len] - - #print(decoded, decoded.shape,'decoded') - - output = [] - for d in decoded: - # Convert the predicted indices to the corresponding chars. - d = tf.strings.reduce_join(self.num_to_char(d)) - d = d.numpy().decode("utf-8") - output.append(d) - return output - - - def distortion_free_resize(self, image, img_size): - w, h = img_size - image = tf.image.resize(image, size=(h, w), preserve_aspect_ratio=True) - - # Check tha amount of padding needed to be done. - pad_height = h - tf.shape(image)[0] - pad_width = w - tf.shape(image)[1] - - # Only necessary if you want to do same amount of padding on both sides. - if pad_height % 2 != 0: - height = pad_height // 2 - pad_height_top = height + 1 - pad_height_bottom = height - else: - pad_height_top = pad_height_bottom = pad_height // 2 - - if pad_width % 2 != 0: - width = pad_width // 2 - pad_width_left = width + 1 - pad_width_right = width - else: - pad_width_left = pad_width_right = pad_width // 2 - - image = tf.pad( - image, - paddings=[ - [pad_height_top, pad_height_bottom], - [pad_width_left, pad_width_right], - [0, 0], - ], - ) - - image = tf.transpose(image, (1, 0, 2)) - image = tf.image.flip_left_right(image) - return image - - def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.22*width) - - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - - if len(peaks_real)>35: - - #peaks_real = peaks_real[(peaks_realwidth1)] - argsort = np.argsort(sum_smoothed[peaks_real])[::-1] - peaks_real_top_six = peaks_real[argsort[:6]] - midpoint = textline_image.shape[1] / 2. - arg_closest = np.argmin(np.abs(peaks_real_top_six - midpoint)) - - #arg_max = np.argmax(sum_smoothed[peaks_real]) - - peaks_final = peaks_real_top_six[arg_closest]#peaks_real[arg_max] - - return peaks_final - else: - return None - - # Function to fit text inside the given area - def fit_text_single_line(self, draw, text, font_path, max_width, max_height): - initial_font_size = 50 - font_size = initial_font_size - while font_size > 10: # Minimum font size - font = ImageFont.truetype(font_path, font_size) - text_bbox = draw.textbbox((0, 0), text, font=font) # Get text bounding box - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] - - if text_width <= max_width and text_height <= max_height: - return font # Return the best-fitting font - - font_size -= 2 # Reduce font size and retry - - return ImageFont.truetype(font_path, 10) # Smallest font fallback - - def return_textlines_split_if_needed(self, textline_image, textline_image_bin): - - split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image) - if split_point: - image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) - image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) - if self.prediction_with_both_of_rgb_and_bin: - image1_bin = textline_image_bin[:, :split_point,:]# image.crop((0, 0, width2, height)) - image2_bin = textline_image_bin[:, split_point:,:]#image.crop((width1, 0, width, height)) - return [image1, image2], [image1_bin, image2_bin] - else: - return [image1, image2], None - else: - return None, None - def preprocess_and_resize_image_for_ocrcnn_model(self, img, image_height, image_width): - ratio = image_height /float(img.shape[0]) - w_ratio = int(ratio * img.shape[1]) - - if w_ratio <= image_width: - width_new = w_ratio - else: - width_new = image_width - - if width_new == 0: - width_new = img.shape[1] - - ##if width_new+32 >= image_width: - ##width_new = width_new - 32 - - ###patch_zero = np.zeros((32, 32, 3))#+255 - ###patch_zero[9:19,8:18,:] = 0 - - - img = resize_image(img, image_height, width_new) - img_fin = np.ones((image_height, image_width, 3))*255 - ###img_fin[:,:32,:] = patch_zero[:,:,:] - ###img_fin[:,32:32+width_new,:] = img[:,:,:] - img_fin[:,:width_new,:] = img[:,:,:] - img_fin = img_fin / 255. - return img_fin - - def get_deskewed_contour_and_bb_and_image(self, contour, image, deskew_angle): - (h_in, w_in) = image.shape[:2] - center = (w_in // 2, h_in // 2) - - rotation_matrix = cv2.getRotationMatrix2D(center, deskew_angle, 1.0) - - cos_angle = abs(rotation_matrix[0, 0]) - sin_angle = abs(rotation_matrix[0, 1]) - new_w = int((h_in * sin_angle) + (w_in * cos_angle)) - new_h = int((h_in * cos_angle) + (w_in * sin_angle)) - - rotation_matrix[0, 2] += (new_w / 2) - center[0] - rotation_matrix[1, 2] += (new_h / 2) - center[1] - - deskewed_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h)) - - contour_points = np.array(contour, dtype=np.float32) - transformed_points = cv2.transform(np.array([contour_points]), rotation_matrix)[0] - - x, y, w, h = cv2.boundingRect(np.array(transformed_points, dtype=np.int32)) - cropped_textline = deskewed_image[y:y+h, x:x+w] - - return cropped_textline - - def rotate_image_with_padding(self, image, angle, border_value=(0,0,0)): - # Get image dimensions - (h, w) = image.shape[:2] - - # Calculate the center of the image - center = (w // 2, h // 2) - - # Get the rotation matrix - rotation_matrix = cv2.getRotationMatrix2D(center, angle, 1.0) - - # Compute the new bounding dimensions - cos = abs(rotation_matrix[0, 0]) - sin = abs(rotation_matrix[0, 1]) - new_w = int((h * sin) + (w * cos)) - new_h = int((h * cos) + (w * sin)) - - # Adjust the rotation matrix to account for translation - rotation_matrix[0, 2] += (new_w / 2) - center[0] - rotation_matrix[1, 2] += (new_h / 2) - center[1] - - # Perform the rotation - rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=border_value) - - return rotated_image - - def get_orientation_moments(self, contour): - moments = cv2.moments(contour) - if moments["mu20"] - moments["mu02"] == 0: # Avoid division by zero - return 90 if moments["mu11"] > 0 else -90 - else: - angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) - return np.degrees(angle) # Convert radians to degrees - - - def get_orientation_moments_of_mask(self, mask): - mask=mask.astype('uint8') - contours, _ = cv2.findContours(mask[:,:,0], cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - - largest_contour = max(contours, key=cv2.contourArea) if contours else None - - moments = cv2.moments(largest_contour) - if moments["mu20"] - moments["mu02"] == 0: # Avoid division by zero - return 90 if moments["mu11"] > 0 else -90 - else: - angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) - return np.degrees(angle) # Convert radians to degrees - - def get_contours_and_bounding_boxes(self, mask): - # Find contours in the binary mask - contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - - largest_contour = max(contours, key=cv2.contourArea) if contours else None - - # Get the bounding rectangle for the contour - x, y, w, h = cv2.boundingRect(largest_contour) - #bounding_boxes.append((x, y, w, h)) - - return x, y, w, h - - def return_splitting_point_of_image(self, image_to_spliited): - width = np.shape(image_to_spliited)[1] - height = np.shape(image_to_spliited)[0] - common_window = int(0.03*width) - - width1 = int ( common_window) - width2 = int ( width - common_window ) - - img_sum = np.sum(image_to_spliited[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 1) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - peaks_real = peaks_real[(peaks_realwidth1)] - - arg_sort = np.argsort(sum_smoothed[peaks_real]) - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - - return np.sort(peaks_sort_4) - - def break_curved_line_into_small_pieces_and_then_merge(self, img_curved, mask_curved): - peaks_4 = self.return_splitting_point_of_image(img_curved) - if len(peaks_4)>0: - imgs_tot = [] - - for ind in range(len(peaks_4)+1): - if ind==0: - img = img_curved[:, :peaks_4[ind], :] - mask = mask_curved[:, :peaks_4[ind], :] - elif ind==len(peaks_4): - img = img_curved[:, peaks_4[ind-1]:, :] - mask = mask_curved[:, peaks_4[ind-1]:, :] - else: - img = img_curved[:, peaks_4[ind-1]:peaks_4[ind], :] - mask = mask_curved[:, peaks_4[ind-1]:peaks_4[ind], :] - - or_ma = self.get_orientation_moments_of_mask(mask) - - imgs_tot.append([img, mask, or_ma] ) - - - w_tot_des_list = [] - w_tot_des = 0 - imgs_deskewed_list = [] - for ind in range(len(imgs_tot)): - img_in = imgs_tot[ind][0] - mask_in = imgs_tot[ind][1] - ori_in = imgs_tot[ind][2] - - if abs(ori_in)<45: - img_in_des = self.rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) - mask_in_des = self.rotate_image_with_padding(mask_in, ori_in) - mask_in_des = mask_in_des.astype('uint8') - - #new bounding box - x_n, y_n, w_n, h_n = self.get_contours_and_bounding_boxes(mask_in_des[:,:,0]) - - mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - - w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) - if w_relative==0: - w_relative = img_in_des.shape[1] - img_in_des = resize_image(img_in_des, 32, w_relative) - - - else: - img_in_des = np.copy(img_in) - w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) - if w_relative==0: - w_relative = img_in_des.shape[1] - img_in_des = resize_image(img_in_des, 32, w_relative) - - w_tot_des+=img_in_des.shape[1] - w_tot_des_list.append(img_in_des.shape[1]) - imgs_deskewed_list.append(img_in_des) - - - - - img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 - - w_indexer = 0 - for ind in range(len(w_tot_des_list)): - img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] - w_indexer = w_indexer+w_tot_des_list[ind] - return img_final_deskewed - else: - return img_curved - def run(self): ls_imgs = os.listdir(self.dir_in) @@ -6069,7 +5814,7 @@ class Eynollah_ocr: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) preds = (preds + preds_bin) / 2. - pred_texts = self.decode_batch_predictions(preds) + pred_texts = self.decode_batch_predictions(preds, self.num_to_char) for ib in range(imgs.shape[0]): pred_texts_ib = pred_texts[ib].replace("[UNK]", "") diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py new file mode 100644 index 0000000..44367b6 --- /dev/null +++ b/src/eynollah/utils/utils_ocr.py @@ -0,0 +1,435 @@ +import numpy as np +import cv2 +import tensorflow as tf +from scipy.signal import find_peaks +from scipy.ndimage import gaussian_filter1d +import math +from .resize import resize_image + +def decode_batch_predictions(pred, num_to_char, max_len = 128): + # input_len is the product of the batch size and the + # number of time steps. + input_len = np.ones(pred.shape[0]) * pred.shape[1] + + # Decode CTC predictions using greedy search. + # decoded is a tuple with 2 elements. + decoded = tf.keras.backend.ctc_decode(pred, + input_length = input_len, + beam_width = 100) + # The outputs are in the first element of the tuple. + # Additionally, the first element is actually a list, + # therefore we take the first element of that list as well. + #print(decoded,'decoded') + decoded = decoded[0][0][:, :max_len] + + #print(decoded, decoded.shape,'decoded') + + output = [] + for d in decoded: + # Convert the predicted indices to the corresponding chars. + d = tf.strings.reduce_join(num_to_char(d)) + d = d.numpy().decode("utf-8") + output.append(d) + return output + + +def distortion_free_resize(image, img_size): + w, h = img_size + image = tf.image.resize(image, size=(h, w), preserve_aspect_ratio=True) + + # Check tha amount of padding needed to be done. + pad_height = h - tf.shape(image)[0] + pad_width = w - tf.shape(image)[1] + + # Only necessary if you want to do same amount of padding on both sides. + if pad_height % 2 != 0: + height = pad_height // 2 + pad_height_top = height + 1 + pad_height_bottom = height + else: + pad_height_top = pad_height_bottom = pad_height // 2 + + if pad_width % 2 != 0: + width = pad_width // 2 + pad_width_left = width + 1 + pad_width_right = width + else: + pad_width_left = pad_width_right = pad_width // 2 + + image = tf.pad( + image, + paddings=[ + [pad_height_top, pad_height_bottom], + [pad_width_left, pad_width_right], + [0, 0], + ], + ) + + image = tf.transpose(image, (1, 0, 2)) + image = tf.image.flip_left_right(image) + return image + +def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image): + width = np.shape(textline_image)[1] + height = np.shape(textline_image)[0] + common_window = int(0.22*width) + + width1 = int ( width/2. - common_window ) + width2 = int ( width/2. + common_window ) + + img_sum = np.sum(textline_image[:,:,0], axis=0) + sum_smoothed = gaussian_filter1d(img_sum, 3) + + peaks_real, _ = find_peaks(sum_smoothed, height=0) + + if len(peaks_real)>35: + + #peaks_real = peaks_real[(peaks_realwidth1)] + argsort = np.argsort(sum_smoothed[peaks_real])[::-1] + peaks_real_top_six = peaks_real[argsort[:6]] + midpoint = textline_image.shape[1] / 2. + arg_closest = np.argmin(np.abs(peaks_real_top_six - midpoint)) + + #arg_max = np.argmax(sum_smoothed[peaks_real]) + + peaks_final = peaks_real_top_six[arg_closest]#peaks_real[arg_max] + + return peaks_final + else: + return None + +# Function to fit text inside the given area +def fit_text_single_line(draw, text, font_path, max_width, max_height): + initial_font_size = 50 + font_size = initial_font_size + while font_size > 10: # Minimum font size + font = ImageFont.truetype(font_path, font_size) + text_bbox = draw.textbbox((0, 0), text, font=font) # Get text bounding box + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + + if text_width <= max_width and text_height <= max_height: + return font # Return the best-fitting font + + font_size -= 2 # Reduce font size and retry + + return ImageFont.truetype(font_path, 10) # Smallest font fallback + +def return_textlines_split_if_needed(textline_image, textline_image_bin, prediction_with_both_of_rgb_and_bin=False): + + split_point = return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image) + if split_point: + image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) + image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) + if prediction_with_both_of_rgb_and_bin: + image1_bin = textline_image_bin[:, :split_point,:]# image.crop((0, 0, width2, height)) + image2_bin = textline_image_bin[:, split_point:,:]#image.crop((width1, 0, width, height)) + return [image1, image2], [image1_bin, image2_bin] + else: + return [image1, image2], None + else: + return None, None +def preprocess_and_resize_image_for_ocrcnn_model(img, image_height, image_width): + ratio = image_height /float(img.shape[0]) + w_ratio = int(ratio * img.shape[1]) + + if w_ratio <= image_width: + width_new = w_ratio + else: + width_new = image_width + + if width_new == 0: + width_new = img.shape[1] + + + img = resize_image(img, image_height, width_new) + img_fin = np.ones((image_height, image_width, 3))*255 + + img_fin[:,:width_new,:] = img[:,:,:] + img_fin = img_fin / 255. + return img_fin + +def get_deskewed_contour_and_bb_and_image(contour, image, deskew_angle): + (h_in, w_in) = image.shape[:2] + center = (w_in // 2, h_in // 2) + + rotation_matrix = cv2.getRotationMatrix2D(center, deskew_angle, 1.0) + + cos_angle = abs(rotation_matrix[0, 0]) + sin_angle = abs(rotation_matrix[0, 1]) + new_w = int((h_in * sin_angle) + (w_in * cos_angle)) + new_h = int((h_in * cos_angle) + (w_in * sin_angle)) + + rotation_matrix[0, 2] += (new_w / 2) - center[0] + rotation_matrix[1, 2] += (new_h / 2) - center[1] + + deskewed_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h)) + + contour_points = np.array(contour, dtype=np.float32) + transformed_points = cv2.transform(np.array([contour_points]), rotation_matrix)[0] + + x, y, w, h = cv2.boundingRect(np.array(transformed_points, dtype=np.int32)) + cropped_textline = deskewed_image[y:y+h, x:x+w] + + return cropped_textline + +def rotate_image_with_padding(image, angle, border_value=(0,0,0)): + # Get image dimensions + (h, w) = image.shape[:2] + + # Calculate the center of the image + center = (w // 2, h // 2) + + # Get the rotation matrix + rotation_matrix = cv2.getRotationMatrix2D(center, angle, 1.0) + + # Compute the new bounding dimensions + cos = abs(rotation_matrix[0, 0]) + sin = abs(rotation_matrix[0, 1]) + new_w = int((h * sin) + (w * cos)) + new_h = int((h * cos) + (w * sin)) + + # Adjust the rotation matrix to account for translation + rotation_matrix[0, 2] += (new_w / 2) - center[0] + rotation_matrix[1, 2] += (new_h / 2) - center[1] + + # Perform the rotation + rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=border_value) + + return rotated_image + +def get_orientation_moments(contour): + moments = cv2.moments(contour) + if moments["mu20"] - moments["mu02"] == 0: # Avoid division by zero + return 90 if moments["mu11"] > 0 else -90 + else: + angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) + return np.degrees(angle) # Convert radians to degrees + + +def get_orientation_moments_of_mask(mask): + mask=mask.astype('uint8') + contours, _ = cv2.findContours(mask[:,:,0], cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + largest_contour = max(contours, key=cv2.contourArea) if contours else None + + moments = cv2.moments(largest_contour) + if moments["mu20"] - moments["mu02"] == 0: # Avoid division by zero + return 90 if moments["mu11"] > 0 else -90 + else: + angle = 0.5 * np.arctan2(2 * moments["mu11"], moments["mu20"] - moments["mu02"]) + return np.degrees(angle) # Convert radians to degrees + +def get_contours_and_bounding_boxes(mask): + # Find contours in the binary mask + contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + largest_contour = max(contours, key=cv2.contourArea) if contours else None + + # Get the bounding rectangle for the contour + x, y, w, h = cv2.boundingRect(largest_contour) + #bounding_boxes.append((x, y, w, h)) + + return x, y, w, h + +def return_splitting_point_of_image(image_to_spliited): + width = np.shape(image_to_spliited)[1] + height = np.shape(image_to_spliited)[0] + common_window = int(0.03*width) + + width1 = int ( common_window) + width2 = int ( width - common_window ) + + img_sum = np.sum(image_to_spliited[:,:,0], axis=0) + sum_smoothed = gaussian_filter1d(img_sum, 1) + + peaks_real, _ = find_peaks(sum_smoothed, height=0) + peaks_real = peaks_real[(peaks_realwidth1)] + + arg_sort = np.argsort(sum_smoothed[peaks_real]) + peaks_sort_4 = peaks_real[arg_sort][::-1][:4] + + return np.sort(peaks_sort_4) + +def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved): + peaks_4 = return_splitting_point_of_image(img_curved) + if len(peaks_4)>0: + imgs_tot = [] + + for ind in range(len(peaks_4)+1): + if ind==0: + img = img_curved[:, :peaks_4[ind], :] + mask = mask_curved[:, :peaks_4[ind], :] + elif ind==len(peaks_4): + img = img_curved[:, peaks_4[ind-1]:, :] + mask = mask_curved[:, peaks_4[ind-1]:, :] + else: + img = img_curved[:, peaks_4[ind-1]:peaks_4[ind], :] + mask = mask_curved[:, peaks_4[ind-1]:peaks_4[ind], :] + + or_ma = get_orientation_moments_of_mask(mask) + + imgs_tot.append([img, mask, or_ma] ) + + + w_tot_des_list = [] + w_tot_des = 0 + imgs_deskewed_list = [] + for ind in range(len(imgs_tot)): + img_in = imgs_tot[ind][0] + mask_in = imgs_tot[ind][1] + ori_in = imgs_tot[ind][2] + + if abs(ori_in)<45: + img_in_des = rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) + mask_in_des = rotate_image_with_padding(mask_in, ori_in) + mask_in_des = mask_in_des.astype('uint8') + + #new bounding box + x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_in_des[:,:,0]) + + mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + if w_relative==0: + w_relative = img_in_des.shape[1] + img_in_des = resize_image(img_in_des, 32, w_relative) + + + else: + img_in_des = np.copy(img_in) + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + if w_relative==0: + w_relative = img_in_des.shape[1] + img_in_des = resize_image(img_in_des, 32, w_relative) + + w_tot_des+=img_in_des.shape[1] + w_tot_des_list.append(img_in_des.shape[1]) + imgs_deskewed_list.append(img_in_des) + + + + + img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 + + w_indexer = 0 + for ind in range(len(w_tot_des_list)): + img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] + w_indexer = w_indexer+w_tot_des_list[ind] + return img_final_deskewed + else: + return img_curved + +def return_textline_contour_with_added_box_coordinate(textline_contour, box_ind): + textline_contour[:,0] = textline_contour[:,0] + box_ind[2] + textline_contour[:,1] = textline_contour[:,1] + box_ind[0] + return textline_contour + + +def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, prediction_model, b_s_ocr, num_to_char, textline_light=False, curved_line=False): + max_len = 512 + padding_token = 299 + image_width = 512#max_len * 4 + image_height = 32 + ind_tot = 0 + #cv2.imwrite('./img_out.png', image_page) + ocr_all_textlines = [] + cropped_lines_region_indexer = [] + cropped_lines_meging_indexing = [] + cropped_lines = [] + indexer_text_region = 0 + + for indexing, ind_poly_first in enumerate(all_found_textline_polygons): + #ocr_textline_in_textregion = [] + for indexing2, ind_poly in enumerate(ind_poly_first): + cropped_lines_region_indexer.append(indexer_text_region) + if not (textline_light or curved_line): + ind_poly = copy.deepcopy(ind_poly) + box_ind = all_box_coord[indexing] + + ind_poly = return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) + #print(ind_poly_copy) + ind_poly[ind_poly<0] = 0 + x, y, w, h = cv2.boundingRect(ind_poly) + + w_scaled = w * image_height/float(h) + + mask_poly = np.zeros(image.shape) + + img_poly_on_img = np.copy(image) + + mask_poly = cv2.fillPoly(mask_poly, pts=[ind_poly], color=(1, 1, 1)) + + + + mask_poly = mask_poly[y:y+h, x:x+w, :] + img_crop = img_poly_on_img[y:y+h, x:x+w, :] + + img_crop[mask_poly==0] = 255 + + if w_scaled < 640:#1.5*image_width: + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(0) + else: + splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) + + if splited_images: + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(1) + + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) + + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(-1) + + else: + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(0) + + indexer_text_region+=1 + + + extracted_texts = [] + + n_iterations = math.ceil(len(cropped_lines) / b_s_ocr) + + for i in range(n_iterations): + if i==(n_iterations-1): + n_start = i*b_s_ocr + imgs = cropped_lines[n_start:] + imgs = np.array(imgs) + imgs = imgs.reshape(imgs.shape[0], image_height, image_width, 3) + + + else: + n_start = i*b_s_ocr + n_end = (i+1)*b_s_ocr + imgs = cropped_lines[n_start:n_end] + imgs = np.array(imgs).reshape(b_s_ocr, image_height, image_width, 3) + + + preds = prediction_model.predict(imgs, verbose=0) + + pred_texts = decode_batch_predictions(preds, num_to_char) + + for ib in range(imgs.shape[0]): + pred_texts_ib = pred_texts[ib].replace("[UNK]", "") + extracted_texts.append(pred_texts_ib) + + extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + + extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] + unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) + + ocr_all_textlines = [] + for ind in unique_cropped_lines_region_indexer: + ocr_textline_in_textregion = [] + extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] + for it_ind, text_textline in enumerate(extracted_texts_merged_un): + ocr_textline_in_textregion.append(text_textline) + ocr_all_textlines.append(ocr_textline_in_textregion) + return ocr_all_textlines diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 8cd1c8e..cf0551b 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -168,7 +168,7 @@ class EynollahXmlWriter(): with open(self.output_filename, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines, conf_contours_textregion): + def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines, conf_contours_textregion, skip_layout_reading_order=False): self.logger.debug('enter build_pagexml_no_full_layout') # create the file structure @@ -184,7 +184,7 @@ class EynollahXmlWriter(): for mm in range(len(found_polygons_text_region)): textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord), conf=conf_contours_textregion[mm]), + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord, skip_layout_reading_order), conf=conf_contours_textregion[mm]), ) #textregion.set_conf(conf_contours_textregion[mm]) page.add_TextRegion(textregion) @@ -303,18 +303,28 @@ class EynollahXmlWriter(): return pcgts - def calculate_polygon_coords(self, contour, page_coord): + def calculate_polygon_coords(self, contour, page_coord, skip_layout_reading_order=False): self.logger.debug('enter calculate_polygon_coords') coords = '' for value_bbox in contour: - if len(value_bbox) == 2: - coords += str(int((value_bbox[0] + page_coord[2]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[1] + page_coord[0]) / self.scale_y)) + if skip_layout_reading_order: + if len(value_bbox) == 2: + coords += str(int((value_bbox[0]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[1]) / self.scale_y)) + else: + coords += str(int((value_bbox[0][0]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[0][1]) / self.scale_y)) else: - coords += str(int((value_bbox[0][0] + page_coord[2]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[0][1] + page_coord[0]) / self.scale_y)) + if len(value_bbox) == 2: + coords += str(int((value_bbox[0] + page_coord[2]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[1] + page_coord[0]) / self.scale_y)) + else: + coords += str(int((value_bbox[0][0] + page_coord[2]) / self.scale_x)) + coords += ',' + coords += str(int((value_bbox[0][1] + page_coord[0]) / self.scale_y)) coords=coords + ' ' return coords[:-1] From 0250a6d3d05904ed53cafde596f364500cad8f08 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 23 May 2025 18:06:53 +0200 Subject: [PATCH 145/374] enhancing ocr --- src/eynollah/eynollah.py | 47 ++++++++++++++++++--------------- src/eynollah/utils/utils_ocr.py | 1 + 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 2564150..1b50713 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -85,7 +85,12 @@ from .utils.utils_ocr import ( preprocess_and_resize_image_for_ocrcnn_model, return_textlines_split_if_needed, decode_batch_predictions, - return_rnn_cnn_ocr_of_given_textlines + return_rnn_cnn_ocr_of_given_textlines, + fit_text_single_line, + break_curved_line_into_small_pieces_and_then_merge, + get_orientation_moments, + rotate_image_with_padding, + get_contours_and_bounding_boxes ) from .utils.separate_lines import ( textline_contours_postprocessing, @@ -5421,7 +5426,7 @@ class Eynollah_ocr: cropped_lines.append(resize_image(img_crop, tr_ocr_input_height_and_width, tr_ocr_input_height_and_width) ) cropped_lines_meging_indexing.append(0) else: - splited_images, _ = self.return_textlines_split_if_needed(img_crop, None) + splited_images, _ = return_textlines_split_if_needed(img_crop, None) #print(splited_images) if splited_images: cropped_lines.append(resize_image(splited_images[0], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) @@ -5474,7 +5479,7 @@ class Eynollah_ocr: w_bb = bb_ind[2] h_bb = bb_ind[3] - font = self.fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) + font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) @@ -5607,14 +5612,14 @@ class Eynollah_ocr: #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') if not self.do_not_mask_with_textline_contour: if angle_degrees > 15: - better_des_slope = self.get_orientation_moments(textline_coords) + better_des_slope = get_orientation_moments(textline_coords) - img_crop = self.rotate_image_with_padding(img_crop, better_des_slope ) - mask_poly = self.rotate_image_with_padding(mask_poly, better_des_slope ) + img_crop = rotate_image_with_padding(img_crop, better_des_slope ) + mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) mask_poly = mask_poly.astype('uint8') #new bounding box - x_n, y_n, w_n, h_n = self.get_contours_and_bounding_boxes(mask_poly[:,:,0]) + x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_poly[:,:,0]) mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] @@ -5622,13 +5627,13 @@ class Eynollah_ocr: img_crop[mask_poly==0] = 255 if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 100: - img_crop = self.break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + img_crop = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) #print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii') else: img_crop[mask_poly==0] = 255 if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 100: - img_crop = self.break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + img_crop = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) @@ -5638,7 +5643,7 @@ class Eynollah_ocr: if not self.export_textline_images_and_text: if w_scaled < 640:#1.5*image_width: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) if angle_degrees > 15: cropped_lines_ver_index.append(1) @@ -5647,15 +5652,15 @@ class Eynollah_ocr: cropped_lines_meging_indexing.append(0) if self.prediction_with_both_of_rgb_and_bin: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) else: if self.prediction_with_both_of_rgb_and_bin: - splited_images, splited_images_bin = self.return_textlines_split_if_needed(img_crop, img_crop_bin) + splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, img_crop_bin) else: - splited_images, splited_images_bin = self.return_textlines_split_if_needed(img_crop, None) + splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) if splited_images: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(1) @@ -5664,7 +5669,7 @@ class Eynollah_ocr: else: cropped_lines_ver_index.append(0) - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(-1) @@ -5675,13 +5680,13 @@ class Eynollah_ocr: cropped_lines_ver_index.append(0) if self.prediction_with_both_of_rgb_and_bin: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[0], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[0], image_height, image_width) cropped_lines_bin.append(img_fin) - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[1], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[1], image_height, image_width) cropped_lines_bin.append(img_fin) else: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) @@ -5691,7 +5696,7 @@ class Eynollah_ocr: cropped_lines_ver_index.append(0) if self.prediction_with_both_of_rgb_and_bin: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) if self.export_textline_images_and_text: @@ -5814,7 +5819,7 @@ class Eynollah_ocr: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) preds = (preds + preds_bin) / 2. - pred_texts = self.decode_batch_predictions(preds, self.num_to_char) + pred_texts = decode_batch_predictions(preds, self.num_to_char) for ib in range(imgs.shape[0]): pred_texts_ib = pred_texts[ib].replace("[UNK]", "") @@ -5844,7 +5849,7 @@ class Eynollah_ocr: w_bb = bb_ind[2] h_bb = bb_ind[3] - font = self.fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) + font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 44367b6..339b38a 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -4,6 +4,7 @@ import tensorflow as tf from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d import math +from PIL import Image, ImageDraw, ImageFont from .resize import resize_image def decode_batch_predictions(pred, num_to_char, max_len = 128): From 25e3a2a99f4e585ee73d39e981897062ccd13a1e Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 23 May 2025 18:30:51 +0200 Subject: [PATCH 146/374] visualizing ro for single xml file --- train/generate_gt_for_training.py | 53 +++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 7e7c6a0..9b7f02b 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -252,6 +252,12 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i @main.command() +@click.option( + "--xml_file", + "-xml", + help="xml filename", + type=click.Path(exists=True, dir_okay=False), +) @click.option( "--dir_xml", "-dx", @@ -271,10 +277,14 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i "-dimg", help="directory where the overlayed plots will be written", ) -def visualize_reading_order(dir_xml, dir_out, dir_imgs): - xml_files_ind = os.listdir(dir_xml) - +def visualize_reading_order(xml_file, dir_xml, dir_out, dir_imgs): + assert xml_file or dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" + if dir_xml: + xml_files_ind = os.listdir(dir_xml) + else: + xml_files_ind = [xml_file] + indexer_start= 0#55166 #min_area = 0.0001 @@ -282,8 +292,17 @@ def visualize_reading_order(dir_xml, dir_out, dir_imgs): indexer = 0 #print(ind_xml) #print('########################') - xml_file = os.path.join(dir_xml,ind_xml ) - f_name = ind_xml.split('.')[0] + #xml_file = os.path.join(dir_xml,ind_xml ) + + if dir_xml: + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = Path(ind_xml).stem + else: + xml_file = os.path.join(ind_xml ) + f_name = Path(ind_xml).stem + print(f_name, 'f_name') + + #f_name = ind_xml.split('.')[0] _, _, _, file_name, id_paragraph, id_header,co_text_paragraph,co_text_header,tot_region_ref,x_len, y_len,index_tot_regions,img_poly = read_xml(xml_file) id_all_text = id_paragraph + id_header @@ -373,6 +392,12 @@ def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): @main.command() +@click.option( + "--xml_file", + "-xml", + help="xml filename", + type=click.Path(exists=True, dir_okay=False), +) @click.option( "--dir_xml", "-dx", @@ -392,14 +417,24 @@ def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): "-dimg", help="directory of images where textline segmentation will be overlayed", ) -def visualize_layout_segmentation(dir_xml, dir_out, dir_imgs): - xml_files_ind = os.listdir(dir_xml) +def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): + assert xml_file and dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" + if dir_xml: + xml_files_ind = os.listdir(dir_xml) + else: + xml_files_ind = [xml_file] + for ind_xml in tqdm(xml_files_ind): indexer = 0 #print(ind_xml) #print('########################') - xml_file = os.path.join(dir_xml,ind_xml ) - f_name = Path(ind_xml).stem + if dir_xml: + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = Path(ind_xml).stem + else: + xml_file = os.path.join(ind_xml ) + f_name = Path(ind_xml).stem + print(f_name, 'f_name') img_file_name_with_format = find_format_of_given_filename_in_dir(dir_imgs, f_name) img = cv2.imread(os.path.join(dir_imgs, img_file_name_with_format)) From ba3420b2d8ea1cbca26aac2cc904dd499b893984 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 25 May 2025 01:12:58 +0200 Subject: [PATCH 147/374] Drop capitals are written separately and are not attached to their corresponding text line. The OCR use case also supports single-image input. --- src/eynollah/cli.py | 11 ++++++++-- src/eynollah/eynollah.py | 46 +++++++++++++++++++++++++++++----------- src/eynollah/writer.py | 8 +++---- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index cd56833..0c18b2c 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -331,6 +331,12 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ @main.command() +@click.option( + "--image", + "-i", + help="image filename", + type=click.Path(exists=True, dir_okay=False), +) @click.option( "--dir_in", "-di", @@ -415,7 +421,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, log_level): +def ocr(image, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) @@ -426,8 +432,9 @@ def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, ex assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit" assert not export_textline_images_and_text or not draw_texts_on_image, "Exporting textline and text -etit can not be set alongside draw text on image -dtoi" assert not export_textline_images_and_text or not prediction_with_both_of_rgb_and_bin, "Exporting textline and text -etit can not be set alongside prediction with both rgb and bin -brb" - + assert (bool(image) ^ bool(dir_in)), "Either -i (single image) or -di (directory) must be provided, but not both." eynollah_ocr = Eynollah_ocr( + image_filename=image, dir_xmls=dir_xmls, dir_out_image_text=dir_out_image_text, dir_in=dir_in, diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 1b50713..aa38274 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5134,10 +5134,10 @@ class Eynollah: pixel_img = 4 polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img) - all_found_textline_polygons = adhere_drop_capital_region_into_corresponding_textline( - text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, - all_box_coord, all_box_coord_h, all_found_textline_polygons, all_found_textline_polygons_h, - kernel=KERNEL, curved_line=self.curved_line, textline_light=self.textline_light) + ##all_found_textline_polygons = adhere_drop_capital_region_into_corresponding_textline( + ##text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, + ##all_box_coord, all_box_coord_h, all_found_textline_polygons, all_found_textline_polygons_h, + ##kernel=KERNEL, curved_line=self.curved_line, textline_light=self.textline_light) if not self.reading_order_machine_based: pixel_seps = 6 @@ -5299,6 +5299,7 @@ class Eynollah_ocr: dir_models, dir_xmls=None, dir_in=None, + image_filename=None, dir_in_bin=None, dir_out=None, dir_out_image_text=None, @@ -5312,6 +5313,7 @@ class Eynollah_ocr: logger=None, ): self.dir_in = dir_in + self.image_filename = image_filename self.dir_in_bin = dir_in_bin self.dir_out = dir_out self.dir_xmls = dir_xmls @@ -5363,13 +5365,20 @@ class Eynollah_ocr: ) def run(self): - ls_imgs = os.listdir(self.dir_in) + if self.dir_in: + ls_imgs = os.listdir(self.dir_in) + else: + ls_imgs = [self.image_filename] if self.tr_ocr: tr_ocr_input_height_and_width = 384 for ind_img in ls_imgs: - file_name = Path(ind_img).stem - dir_img = os.path.join(self.dir_in, ind_img) + if self.dir_in: + file_name = Path(ind_img).stem + dir_img = os.path.join(self.dir_in, ind_img) + else: + file_name = Path(self.image_filename).stem + dir_img = self.image_filename dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') img = cv2.imread(dir_img) @@ -5541,8 +5550,15 @@ class Eynollah_ocr: img_size=(image_width, image_height) for ind_img in ls_imgs: - file_name = Path(ind_img).stem - dir_img = os.path.join(self.dir_in, ind_img) + if self.dir_in: + file_name = Path(ind_img).stem + dir_img = os.path.join(self.dir_in, ind_img) + else: + file_name = Path(self.image_filename).stem + dir_img = self.image_filename + + #file_name = Path(ind_img).stem + #dir_img = os.path.join(self.dir_in, ind_img) dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') img = cv2.imread(dir_img) @@ -5576,6 +5592,7 @@ class Eynollah_ocr: indexer_text_region = 0 indexer_textlines = 0 for nn in root1.iter(region_tags): + type_textregion = nn.attrib['type'] for child_textregion in nn: if child_textregion.tag.endswith("TextLine"): for child_textlines in child_textregion: @@ -5589,7 +5606,9 @@ class Eynollah_ocr: angle_radians = math.atan2(h, w) # Convert to degrees angle_degrees = math.degrees(angle_radians) - + if type_textregion=='drop-capital': + angle_degrees = 0 + if self.draw_texts_on_image: total_bb_coordinates.append([x,y,w,h]) @@ -5632,8 +5651,11 @@ class Eynollah_ocr: #print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii') else: img_crop[mask_poly==0] = 255 - if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 100: - img_crop = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + if type_textregion=='drop-capital': + pass + else: + if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 100: + img_crop = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index cf0551b..f07abf6 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -283,14 +283,14 @@ class EynollahXmlWriter(): Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) page.add_TextRegion(marginal) self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) - + for mm in range(len(found_polygons_drop_capitals)): dropcapital = TextRegionType(id=counter.next_region_id, type_='drop-capital', Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_drop_capitals[mm], page_coord))) page.add_TextRegion(dropcapital) - ###all_box_coord_drop = None - ###slopes_drop = None - ###self.serialize_lines_in_dropcapital(dropcapital, [found_polygons_drop_capitals[mm]], mm, page_coord, all_box_coord_drop, slopes_drop, counter, ocr_all_textlines_textregion=None) + all_box_coord_drop = None + slopes_drop = None + self.serialize_lines_in_dropcapital(dropcapital, [found_polygons_drop_capitals[mm]], mm, page_coord, all_box_coord_drop, slopes_drop, counter, ocr_all_textlines_textregion=None) for mm in range(len(found_polygons_text_region_img)): page.add_ImageRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_img[mm], page_coord)))) From b18691f96a5f67e5fda1e6b46d1a399bf20fe858 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 25 May 2025 03:33:54 +0200 Subject: [PATCH 148/374] rnn ocr for all layout textregion types --- src/eynollah/eynollah.py | 41 ++++++++++++++++++++++++++-------------- src/eynollah/writer.py | 31 ++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index aa38274..0ee3d14 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4715,11 +4715,10 @@ class Eynollah: if self.extract_only_images: text_regions_p_1, erosion_hurts, polygons_lines_xml, polygons_of_images, image_page, page_coord, cont_page = \ self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) - ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], polygons_of_images, [], [], [], [], [], - cont_page, [], [], ocr_all_textlines, []) + cont_page, [], []) if self.plotter: self.plotter.write_images_into_directory(polygons_of_images, image_page) return pcgts @@ -4772,7 +4771,7 @@ class Eynollah: cont_page, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions, self.skip_layout_and_reading_order) + cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines=ocr_all_textlines, conf_contours_textregion=conf_contours_textregions, skip_layout_reading_order=self.skip_layout_and_reading_order) return pcgts #print("text region early -1 in %.1fs", time.time() - t0) @@ -4822,10 +4821,9 @@ class Eynollah: if not num_col: self.logger.info("No columns detected, outputting an empty PAGE-XML") - ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], [], [], [], [], [], [], - cont_page, [], [], ocr_all_textlines, []) + cont_page, [], []) return pcgts #print("text region early in %.1fs", time.time() - t0) @@ -5004,13 +5002,13 @@ class Eynollah: [], [], page_coord, [], [], [], [], [], [], polygons_of_images, contours_tables, [], polygons_of_marginals, empty_marginals, empty_marginals, [], [], [], - cont_page, polygons_lines_xml, [], [], []) + cont_page, polygons_lines_xml) else: pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], polygons_of_images, polygons_of_marginals, empty_marginals, empty_marginals, [], [], - cont_page, polygons_lines_xml, contours_tables, [], []) + cont_page, polygons_lines_xml, contours_tables) return pcgts @@ -5196,16 +5194,28 @@ class Eynollah: contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) self.logger.info("detection of reading order took %.1fs", time.time() - t_order) - if self.ocr: - ocr_all_textlines = [] + if self.ocr and not self.tr: + gc.collect() + if len(all_found_textline_polygons)>0: + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + if all_found_textline_polygons_marginals and len(all_found_textline_polygons_marginals)>0: + ocr_all_textlines_marginals = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + + if all_found_textline_polygons_h and len(all_found_textline_polygons)>0: + ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_h, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + if polygons_of_drop_capitals and len(polygons_of_drop_capitals)>0: + ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines(image_page, polygons_of_drop_capitals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines = None + ocr_all_textlines_marginals = None + ocr_all_textlines_h = None + ocr_all_textlines_drop = None pcgts = self.writer.build_pagexml_full_layout( contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, - cont_page, polygons_lines_xml, ocr_all_textlines, conf_contours_textregions, conf_contours_textregions_h) + cont_page, polygons_lines_xml, ocr_all_textlines, ocr_all_textlines_h, ocr_all_textlines_marginals, ocr_all_textlines_drop, conf_contours_textregions, conf_contours_textregions_h) return pcgts contours_only_text_parent_h = None @@ -5278,18 +5288,21 @@ class Eynollah: elif self.ocr and not self.tr: gc.collect() - ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - + if len(all_found_textline_polygons)>0: + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + if all_found_textline_polygons_marginals and len(all_found_textline_polygons_marginals)>0: + ocr_all_textlines_marginals = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines = None - #print(ocr_all_textlines) + ocr_all_textlines_marginals = None self.logger.info("detection of reading order took %.1fs", time.time() - t_order) + pcgts = self.writer.build_pagexml_no_full_layout( txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, ocr_all_textlines_marginals, conf_contours_textregions) return pcgts diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index f07abf6..085ee6f 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -56,10 +56,12 @@ class EynollahXmlWriter(): points_page_print = points_page_print + ' ' return points_page_print[:-1] - def serialize_lines_in_marginal(self, marginal_region, all_found_textline_polygons_marginals, marginal_idx, page_coord, all_box_coord_marginals, slopes_marginals, counter): + def serialize_lines_in_marginal(self, marginal_region, all_found_textline_polygons_marginals, marginal_idx, page_coord, all_box_coord_marginals, slopes_marginals, counter, ocr_all_textlines_textregion): for j in range(len(all_found_textline_polygons_marginals[marginal_idx])): coords = CoordsType() textline = TextLineType(id=counter.next_line_id, Coords=coords) + if ocr_all_textlines_textregion: + textline.set_TextEquiv( [ TextEquivType(Unicode=ocr_all_textlines_textregion[j]) ] ) marginal_region.add_TextLine(textline) marginal_region.set_orientation(-slopes_marginals[marginal_idx]) points_co = '' @@ -168,7 +170,7 @@ class EynollahXmlWriter(): with open(self.output_filename, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines, conf_contours_textregion, skip_layout_reading_order=False): + def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines=None, ocr_all_textlines_marginals=None, conf_contours_textregion=None, skip_layout_reading_order=False): self.logger.debug('enter build_pagexml_no_full_layout') # create the file structure @@ -198,7 +200,12 @@ class EynollahXmlWriter(): marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) page.add_TextRegion(marginal) - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) + if ocr_all_textlines_marginals: + ocr_textlines = ocr_all_textlines_marginals[mm] + else: + ocr_textlines = None + + self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter, ocr_textlines) for mm in range(len(found_polygons_text_region_img)): img_region = ImageRegionType(id=counter.next_region_id, Coords=CoordsType()) @@ -242,7 +249,7 @@ class EynollahXmlWriter(): return pcgts - def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, ocr_all_textlines, conf_contours_textregion, conf_contours_textregion_h): + def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals=None, ocr_all_textlines_drop=None, conf_contours_textregion=None, conf_contours_textregion_h=None): self.logger.debug('enter build_pagexml_full_layout') # create the file structure @@ -272,8 +279,8 @@ class EynollahXmlWriter(): Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_h[mm], page_coord))) page.add_TextRegion(textregion) - if ocr_all_textlines: - ocr_textlines = ocr_all_textlines[mm] + if ocr_all_textlines_h: + ocr_textlines = ocr_all_textlines_h[mm] else: ocr_textlines = None self.serialize_lines_in_region(textregion, all_found_textline_polygons_h, mm, page_coord, all_box_coord_h, slopes_h, counter, ocr_textlines) @@ -282,7 +289,11 @@ class EynollahXmlWriter(): marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) page.add_TextRegion(marginal) - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) + if ocr_all_textlines_marginals: + ocr_textlines = ocr_all_textlines_marginals[mm] + else: + ocr_textlines = None + self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter, ocr_textlines) for mm in range(len(found_polygons_drop_capitals)): dropcapital = TextRegionType(id=counter.next_region_id, type_='drop-capital', @@ -290,7 +301,11 @@ class EynollahXmlWriter(): page.add_TextRegion(dropcapital) all_box_coord_drop = None slopes_drop = None - self.serialize_lines_in_dropcapital(dropcapital, [found_polygons_drop_capitals[mm]], mm, page_coord, all_box_coord_drop, slopes_drop, counter, ocr_all_textlines_textregion=None) + if ocr_all_textlines_drop: + ocr_textlines = ocr_all_textlines_drop[mm] + else: + ocr_textlines = None + self.serialize_lines_in_dropcapital(dropcapital, [found_polygons_drop_capitals[mm]], mm, page_coord, all_box_coord_drop, slopes_drop, counter, ocr_all_textlines_textregion=ocr_textlines) for mm in range(len(found_polygons_text_region_img)): page.add_ImageRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_img[mm], page_coord)))) From 31d9fa0c80191786de97cca0cd7be3d0f7248140 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 25 May 2025 21:44:36 +0200 Subject: [PATCH 149/374] strings alignment function is added + new changes needed for prediction with both bin and rgb inputs is implemented --- requirements.txt | 1 + src/eynollah/eynollah.py | 78 +++++++++++++++++++++++++++------ src/eynollah/utils/utils_ocr.py | 47 +++++++++++++++++--- 3 files changed, 107 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index aeffd47..4bc0c6a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ tensorflow < 2.13 numba <= 0.58.1 scikit-image loky +biopython diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0ee3d14..1f79995 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5647,6 +5647,10 @@ class Eynollah_ocr: better_des_slope = get_orientation_moments(textline_coords) img_crop = rotate_image_with_padding(img_crop, better_des_slope ) + + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) + mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) mask_poly = mask_poly.astype('uint8') @@ -5655,26 +5659,35 @@ class Eynollah_ocr: mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] - + img_crop[mask_poly==0] = 255 + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_crop_bin[mask_poly==0] = 255 + if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 100: - img_crop = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) - - #print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii') + if self.prediction_with_both_of_rgb_and_bin: + img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) + else: + img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + + else: img_crop[mask_poly==0] = 255 + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin[mask_poly==0] = 255 if type_textregion=='drop-capital': pass else: if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 100: - img_crop = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + if self.prediction_with_both_of_rgb_and_bin: + img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) + else: + img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) - - - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin[mask_poly==0] = 255 + if not self.export_textline_images_and_text: if w_scaled < 640:#1.5*image_width: @@ -5796,6 +5809,14 @@ class Eynollah_ocr: imgs_bin = cropped_lines_bin[n_start:] imgs_bin = np.array(imgs_bin) imgs_bin = imgs_bin.reshape(imgs_bin.shape[0], image_height, image_width, 3) + + if len(indices_ver)>0: + imgs_bin_ver_flipped = imgs_bin[indices_ver, : ,: ,:] + imgs_bin_ver_flipped = imgs_bin_ver_flipped[:,::-1,::-1,:] + #print(imgs_ver_flipped, 'imgs_ver_flipped') + + else: + imgs_bin_ver_flipped = None else: n_start = i*self.b_s n_end = (i+1)*self.b_s @@ -5817,22 +5838,25 @@ class Eynollah_ocr: if self.prediction_with_both_of_rgb_and_bin: imgs_bin = cropped_lines_bin[n_start:n_end] imgs_bin = np.array(imgs_bin).reshape(self.b_s, image_height, image_width, 3) + + + if len(indices_ver)>0: + imgs_bin_ver_flipped = imgs_bin[indices_ver, : ,: ,:] + imgs_bin_ver_flipped = imgs_bin_ver_flipped[:,::-1,::-1,:] + #print(imgs_ver_flipped, 'imgs_ver_flipped') + else: + imgs_bin_ver_flipped = None preds = self.prediction_model.predict(imgs, verbose=0) if len(indices_ver)>0: - #cv2.imwrite('flipped.png', (imgs_ver_flipped[0, :,:,:]*255).astype('uint8')) - #cv2.imwrite('original.png', (imgs[0, :,:,:]*255).astype('uint8')) - #sys.exit() - #print(imgs_ver_flipped.shape, 'imgs_ver_flipped.shape') preds_flipped = self.prediction_model.predict(imgs_ver_flipped, verbose=0) preds_max_fliped = np.max(preds_flipped, axis=2 ) preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=256 masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) masked_means_flipped[np.isnan(masked_means_flipped)] = 0 - #print(masked_means_flipped, 'masked_means_flipped') preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) @@ -5852,6 +5876,32 @@ class Eynollah_ocr: preds[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] if self.prediction_with_both_of_rgb_and_bin: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) + + if len(indices_ver)>0: + preds_flipped = self.prediction_model.predict(imgs_bin_ver_flipped, verbose=0) + preds_max_fliped = np.max(preds_flipped, axis=2 ) + preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) + pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=256 + masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) + masked_means_flipped[np.isnan(masked_means_flipped)] = 0 + + preds_max = np.max(preds, axis=2 ) + preds_max_args = np.argmax(preds, axis=2 ) + pred_max_not_unk_mask_bool = preds_max_args[:,:]!=256 + + masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) + masked_means[np.isnan(masked_means)] = 0 + + masked_means_ver = masked_means[indices_ver] + #print(masked_means_ver, 'pred_max_not_unk') + + indices_where_flipped_conf_value_is_higher = np.where(masked_means_flipped > masked_means_ver)[0] + + #print(indices_where_flipped_conf_value_is_higher, 'indices_where_flipped_conf_value_is_higher') + if len(indices_where_flipped_conf_value_is_higher)>0: + indices_to_be_replaced = indices_ver[indices_where_flipped_conf_value_is_higher] + preds_bin[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] + preds = (preds + preds_bin) / 2. pred_texts = decode_batch_predictions(preds, self.num_to_char) diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 339b38a..524e7ce 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -5,6 +5,7 @@ from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d import math from PIL import Image, ImageDraw, ImageFont +from Bio import pairwise2 from .resize import resize_image def decode_batch_predictions(pred, num_to_char, max_len = 128): @@ -252,7 +253,7 @@ def return_splitting_point_of_image(image_to_spliited): return np.sort(peaks_sort_4) -def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved): +def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved, img_bin_curved=None): peaks_4 = return_splitting_point_of_image(img_curved) if len(peaks_4)>0: imgs_tot = [] @@ -260,29 +261,44 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved): for ind in range(len(peaks_4)+1): if ind==0: img = img_curved[:, :peaks_4[ind], :] + if img_bin_curved: + img_bin = img_curved_bin[:, :peaks_4[ind], :] mask = mask_curved[:, :peaks_4[ind], :] elif ind==len(peaks_4): img = img_curved[:, peaks_4[ind-1]:, :] + if img_bin_curved: + img_bin = img_curved_bin[:, peaks_4[ind-1]:, :] mask = mask_curved[:, peaks_4[ind-1]:, :] else: img = img_curved[:, peaks_4[ind-1]:peaks_4[ind], :] + if img_bin_curved: + img_bin = img_curved_bin[:, peaks_4[ind-1]:peaks_4[ind], :] mask = mask_curved[:, peaks_4[ind-1]:peaks_4[ind], :] or_ma = get_orientation_moments_of_mask(mask) - - imgs_tot.append([img, mask, or_ma] ) + + if img_bin_curved: + imgs_tot.append([img, mask, or_ma, img_bin] ) + else: + imgs_tot.append([img, mask, or_ma] ) w_tot_des_list = [] w_tot_des = 0 imgs_deskewed_list = [] + imgs_bin_deskewed_list = [] + for ind in range(len(imgs_tot)): img_in = imgs_tot[ind][0] mask_in = imgs_tot[ind][1] ori_in = imgs_tot[ind][2] + if img_bin_curved: + img_bin_in = imgs_tot[ind][3] if abs(ori_in)<45: img_in_des = rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) + if img_bin_curved: + img_bin_in_des = rotate_image_with_padding(img_bin_in, ori_in, border_value=(255,255,255) ) mask_in_des = rotate_image_with_padding(mask_in, ori_in) mask_in_des = mask_in_des.astype('uint8') @@ -291,36 +307,52 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved): mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + if img_bin_curved: + img_bin_in_des = img_bin_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) if w_relative==0: w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) + if img_bin_curved: + img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) else: img_in_des = np.copy(img_in) + if img_bin_curved: + img_bin_in_des = np.copy(img_bin_in) w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) if w_relative==0: w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) + if img_bin_curved: + img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) w_tot_des+=img_in_des.shape[1] w_tot_des_list.append(img_in_des.shape[1]) imgs_deskewed_list.append(img_in_des) + if img_bin_curved: + imgs_bin_deskewed_list.append(img_bin_in_des) img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 + if img_bin_curved: + img_bin_final_deskewed = np.zeros((32, w_tot_des, 3))+255 + else: + img_bin_final_deskewed = None w_indexer = 0 for ind in range(len(w_tot_des_list)): img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] + if img_bin_curved: + img_bin_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_bin_deskewed_list[ind][:,:,:] w_indexer = w_indexer+w_tot_des_list[ind] - return img_final_deskewed + return img_final_deskewed, img_bin_final_deskewed else: - return img_curved + return img_curved, img_bin_curved def return_textline_contour_with_added_box_coordinate(textline_contour, box_ind): textline_contour[:,0] = textline_contour[:,0] + box_ind[2] @@ -434,3 +466,8 @@ def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, pr ocr_textline_in_textregion.append(text_textline) ocr_all_textlines.append(ocr_textline_in_textregion) return ocr_all_textlines + +def biopython_align(str1, str2): + alignments = pairwise2.align.globalms(str1, str2, 2, -1, -2, -2) + best_alignment = alignments[0] # Get the best alignment + return best_alignment.seqA, best_alignment.seqB From 03f52e7a467869d6476de6632411e4e93320bf14 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 27 May 2025 23:45:22 +0200 Subject: [PATCH 150/374] updating ocr --- src/eynollah/cli.py | 10 ++++-- src/eynollah/eynollah.py | 24 ++++++++++++-- src/eynollah/utils/utils_ocr.py | 55 +++++++++++++++++---------------- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 0c18b2c..2d0d6f9 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -337,6 +337,12 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="image filename", type=click.Path(exists=True, dir_okay=False), ) +@click.option( + "--overwrite", + "-O", + help="overwrite (instead of skipping) if output xml exists", + is_flag=True, +) @click.option( "--dir_in", "-di", @@ -421,7 +427,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(image, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, log_level): +def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) @@ -449,7 +455,7 @@ def ocr(image, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ batch_size=batch_size, pref_of_dataset=dataset_abbrevation, ) - eynollah_ocr.run() + eynollah_ocr.run(overwrite=overwrite) if __name__ == "__main__": main() diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 1f79995..efa1dde 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5338,6 +5338,8 @@ class Eynollah_ocr: self.dir_out_image_text = dir_out_image_text self.prediction_with_both_of_rgb_and_bin = prediction_with_both_of_rgb_and_bin self.pref_of_dataset = pref_of_dataset + self.logger = logger if logger else getLogger('eynollah') + if not export_textline_images_and_text: if tr_ocr: self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") @@ -5351,7 +5353,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_750000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_1075000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5377,7 +5379,7 @@ class Eynollah_ocr: vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True ) - def run(self): + def run(self, overwrite : bool = False): if self.dir_in: ls_imgs = os.listdir(self.dir_in) else: @@ -5394,6 +5396,14 @@ class Eynollah_ocr: dir_img = self.image_filename dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') + + if os.path.exists(out_file_ocr): + if overwrite: + self.logger.warning("will overwrite existing output file '%s'", out_file_ocr) + else: + self.logger.warning("will skip input for existing output file '%s'", out_file_ocr) + continue + img = cv2.imread(dir_img) if self.draw_texts_on_image: @@ -5574,6 +5584,14 @@ class Eynollah_ocr: #dir_img = os.path.join(self.dir_in, ind_img) dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') + + if os.path.exists(out_file_ocr): + if overwrite: + self.logger.warning("will overwrite existing output file '%s'", out_file_ocr) + else: + self.logger.warning("will skip input for existing output file '%s'", out_file_ocr) + continue + img = cv2.imread(dir_img) if self.prediction_with_both_of_rgb_and_bin: cropped_lines_bin = [] @@ -5704,7 +5722,7 @@ class Eynollah_ocr: cropped_lines_bin.append(img_fin) else: if self.prediction_with_both_of_rgb_and_bin: - splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, img_crop_bin) + splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, img_crop_bin, prediction_with_both_of_rgb_and_bin=self.prediction_with_both_of_rgb_and_bin) else: splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) if splited_images: diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 524e7ce..9ef344a 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -74,32 +74,24 @@ def distortion_free_resize(image, img_size): def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image): width = np.shape(textline_image)[1] height = np.shape(textline_image)[0] - common_window = int(0.22*width) + common_window = int(0.06*width) width1 = int ( width/2. - common_window ) width2 = int ( width/2. + common_window ) - + img_sum = np.sum(textline_image[:,:,0], axis=0) sum_smoothed = gaussian_filter1d(img_sum, 3) - + peaks_real, _ = find_peaks(sum_smoothed, height=0) - - if len(peaks_real)>35: + if len(peaks_real)>70: - #peaks_real = peaks_real[(peaks_realwidth1)] - argsort = np.argsort(sum_smoothed[peaks_real])[::-1] - peaks_real_top_six = peaks_real[argsort[:6]] - midpoint = textline_image.shape[1] / 2. - arg_closest = np.argmin(np.abs(peaks_real_top_six - midpoint)) + peaks_real = peaks_real[(peaks_realwidth1)] - #arg_max = np.argmax(sum_smoothed[peaks_real]) - - peaks_final = peaks_real_top_six[arg_closest]#peaks_real[arg_max] - + arg_max = np.argmax(sum_smoothed[peaks_real]) + peaks_final = peaks_real[arg_max] return peaks_final else: return None - # Function to fit text inside the given area def fit_text_single_line(draw, text, font_path, max_width, max_height): initial_font_size = 50 @@ -305,17 +297,28 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved, #new bounding box x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_in_des[:,:,0]) - mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - if img_bin_curved: - img_bin_in_des = img_bin_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - - w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) - if w_relative==0: - w_relative = img_in_des.shape[1] - img_in_des = resize_image(img_in_des, 32, w_relative) - if img_bin_curved: - img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) + if w_n==0 or h_n==0: + img_in_des = np.copy(img_in) + if img_bin_curved: + img_bin_in_des = np.copy(img_bin_in) + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + if w_relative==0: + w_relative = img_in_des.shape[1] + img_in_des = resize_image(img_in_des, 32, w_relative) + if img_bin_curved: + img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) + else: + mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + if img_bin_curved: + img_bin_in_des = img_bin_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] + + w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) + if w_relative==0: + w_relative = img_in_des.shape[1] + img_in_des = resize_image(img_in_des, 32, w_relative) + if img_bin_curved: + img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) else: From 1e7cecfcf9534c93b24c11fc7b988a0bd5230a4f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 28 May 2025 01:17:21 +0200 Subject: [PATCH 151/374] updating ocr --- src/eynollah/eynollah.py | 2 +- src/eynollah/utils/utils_ocr.py | 36 ++++++++++++++++----------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index efa1dde..0a9248e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5353,7 +5353,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_1075000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_1150000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 9ef344a..aa1efa6 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -253,23 +253,23 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved, for ind in range(len(peaks_4)+1): if ind==0: img = img_curved[:, :peaks_4[ind], :] - if img_bin_curved: - img_bin = img_curved_bin[:, :peaks_4[ind], :] + if img_bin_curved is not None: + img_bin = img_bin_curved[:, :peaks_4[ind], :] mask = mask_curved[:, :peaks_4[ind], :] elif ind==len(peaks_4): img = img_curved[:, peaks_4[ind-1]:, :] - if img_bin_curved: - img_bin = img_curved_bin[:, peaks_4[ind-1]:, :] + if img_bin_curved is not None: + img_bin = img_bin_curved[:, peaks_4[ind-1]:, :] mask = mask_curved[:, peaks_4[ind-1]:, :] else: img = img_curved[:, peaks_4[ind-1]:peaks_4[ind], :] - if img_bin_curved: - img_bin = img_curved_bin[:, peaks_4[ind-1]:peaks_4[ind], :] + if img_bin_curved is not None: + img_bin = img_bin_curved[:, peaks_4[ind-1]:peaks_4[ind], :] mask = mask_curved[:, peaks_4[ind-1]:peaks_4[ind], :] or_ma = get_orientation_moments_of_mask(mask) - if img_bin_curved: + if img_bin_curved is not None: imgs_tot.append([img, mask, or_ma, img_bin] ) else: imgs_tot.append([img, mask, or_ma] ) @@ -284,12 +284,12 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved, img_in = imgs_tot[ind][0] mask_in = imgs_tot[ind][1] ori_in = imgs_tot[ind][2] - if img_bin_curved: + if img_bin_curved is not None: img_bin_in = imgs_tot[ind][3] if abs(ori_in)<45: img_in_des = rotate_image_with_padding(img_in, ori_in, border_value=(255,255,255) ) - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = rotate_image_with_padding(img_bin_in, ori_in, border_value=(255,255,255) ) mask_in_des = rotate_image_with_padding(mask_in, ori_in) mask_in_des = mask_in_des.astype('uint8') @@ -299,50 +299,50 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved, if w_n==0 or h_n==0: img_in_des = np.copy(img_in) - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = np.copy(img_bin_in) w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) if w_relative==0: w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) else: mask_in_des = mask_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] img_in_des = img_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = img_bin_in_des[y_n:y_n+h_n, x_n:x_n+w_n, :] w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) if w_relative==0: w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) else: img_in_des = np.copy(img_in) - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = np.copy(img_bin_in) w_relative = int(32 * img_in_des.shape[1]/float(img_in_des.shape[0]) ) if w_relative==0: w_relative = img_in_des.shape[1] img_in_des = resize_image(img_in_des, 32, w_relative) - if img_bin_curved: + if img_bin_curved is not None: img_bin_in_des = resize_image(img_bin_in_des, 32, w_relative) w_tot_des+=img_in_des.shape[1] w_tot_des_list.append(img_in_des.shape[1]) imgs_deskewed_list.append(img_in_des) - if img_bin_curved: + if img_bin_curved is not None: imgs_bin_deskewed_list.append(img_bin_in_des) img_final_deskewed = np.zeros((32, w_tot_des, 3))+255 - if img_bin_curved: + if img_bin_curved is not None: img_bin_final_deskewed = np.zeros((32, w_tot_des, 3))+255 else: img_bin_final_deskewed = None @@ -350,7 +350,7 @@ def break_curved_line_into_small_pieces_and_then_merge(img_curved, mask_curved, w_indexer = 0 for ind in range(len(w_tot_des_list)): img_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_deskewed_list[ind][:,:,:] - if img_bin_curved: + if img_bin_curved is not None: img_bin_final_deskewed[:,w_indexer:w_indexer+w_tot_des_list[ind],:] = imgs_bin_deskewed_list[ind][:,:,:] w_indexer = w_indexer+w_tot_des_list[ind] return img_final_deskewed, img_bin_final_deskewed From df903aa1b45f43a44eb324e71b5b911763a4d47c Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sat, 31 May 2025 01:09:14 +0200 Subject: [PATCH 152/374] Parametrize OCR for handling curved lines --- src/eynollah/eynollah.py | 10 +++++----- src/eynollah/utils/utils_ocr.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0a9248e..6c00329 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5353,7 +5353,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_1150000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_1225000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5642,7 +5642,7 @@ class Eynollah_ocr: if self.draw_texts_on_image: total_bb_coordinates.append([x,y,w,h]) - + w_scaled = w * image_height/float(h) img_poly_on_img = np.copy(img) @@ -5684,7 +5684,7 @@ class Eynollah_ocr: img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] img_crop_bin[mask_poly==0] = 255 - if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 100: + if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: if self.prediction_with_both_of_rgb_and_bin: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: @@ -5698,7 +5698,7 @@ class Eynollah_ocr: if type_textregion=='drop-capital': pass else: - if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 100: + if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: if self.prediction_with_both_of_rgb_and_bin: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: @@ -5708,7 +5708,7 @@ class Eynollah_ocr: if not self.export_textline_images_and_text: - if w_scaled < 640:#1.5*image_width: + if w_scaled < 530:#640:#1.5*image_width: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) if angle_degrees > 15: diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index aa1efa6..81a8ae1 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -241,7 +241,7 @@ def return_splitting_point_of_image(image_to_spliited): peaks_real = peaks_real[(peaks_realwidth1)] arg_sort = np.argsort(sum_smoothed[peaks_real]) - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] + peaks_sort_4 = peaks_real[arg_sort][::-1][:3] return np.sort(peaks_sort_4) From 3b475915c79ee8c1690349f2d08625ab479eb930 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 1 Jun 2025 15:53:04 +0200 Subject: [PATCH 153/374] image enhancer is integrated --- src/eynollah/cli.py | 69 +++ src/eynollah/eynollah.py | 234 +--------- src/eynollah/image_enhancer.py | 756 +++++++++++++++++++++++++++++++++ 3 files changed, 830 insertions(+), 229 deletions(-) create mode 100644 src/eynollah/image_enhancer.py diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 2d0d6f9..840bc4b 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -3,6 +3,7 @@ import click from ocrd_utils import initLogging, getLevelName, getLogger from eynollah.eynollah import Eynollah, Eynollah_ocr from eynollah.sbb_binarize import SbbBinarizer +from eynollah.image_enhancer import Enhancer @click.group() def main(): @@ -70,6 +71,74 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) +@main.command() +@click.option( + "--image", + "-i", + help="image filename", + type=click.Path(exists=True, dir_okay=False), +) + +@click.option( + "--out", + "-o", + help="directory to write output xml data", + type=click.Path(exists=True, file_okay=False), + required=True, +) +@click.option( + "--overwrite", + "-O", + help="overwrite (instead of skipping) if output xml exists", + is_flag=True, +) +@click.option( + "--dir_in", + "-di", + help="directory of images", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--model", + "-m", + help="directory of models", + type=click.Path(exists=True, file_okay=False), + required=True, +) + +@click.option( + "--num_col_upper", + "-ncu", + help="lower limit of columns in document image", +) +@click.option( + "--num_col_lower", + "-ncl", + help="upper limit of columns in document image", +) +@click.option( + "--log_level", + "-l", + type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), + help="Override log level globally to this", +) + +def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_lower, log_level): + initLogging() + if log_level: + getLogger('enhancement').setLevel(getLevelName(log_level)) + assert image or dir_in, "Either a single image -i or a dir_in -di is required" + enhancer_object = Enhancer( + model, + logger=getLogger('enhancement'), + dir_out=out, + num_col_upper=num_col_upper, + num_col_lower=num_col_lower, + ) + if dir_in: + enhancer_object.run(dir_in=dir_in, overwrite=overwrite) + else: + enhancer_object.run(image_filename=image, overwrite=overwrite) @main.command() @click.option( diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6c00329..cf540d3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3612,25 +3612,12 @@ class Eynollah: inference_bs = 3 - cv2.imwrite('textregions.png', text_regions_p*50) - cv2.imwrite('sep.png', (text_regions_p[:,:]==6)*255) - ver_kernel = np.ones((5, 1), dtype=np.uint8) hor_kernel = np.ones((1, 5), dtype=np.uint8) - - #separators = (text_regions_p[:,:]==6)*1 - #text_regions_p[text_regions_p[:,:]==6] = 0 - #separators = separators.astype('uint8') - - #separators = cv2.erode(separators , hor_kernel, iterations=1) - #text_regions_p[separators[:,:]==1] = 6 - - #cv2.imwrite('sep_new.png', (text_regions_p[:,:]==6)*255) - min_cont_size_to_be_dilated = 10 - if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: cx_conts, cy_conts, x_min_conts, x_max_conts, y_min_conts, y_max_conts, _ = find_new_features_of_contours(contours_only_text_parent) args_cont_located = np.array(range(len(contours_only_text_parent))) @@ -3672,7 +3659,6 @@ class Eynollah: text_regions_p_textregions_dilated = cv2.dilate(text_regions_p_textregions_dilated , ver_kernel, iterations=5) text_regions_p_textregions_dilated[text_regions_p[:,:]>1] = 0 - cv2.imwrite('text_regions_p_textregions_dilated.png', text_regions_p_textregions_dilated*255) contours_only_dilated, hir_on_text_dilated = return_contours_of_image(text_regions_p_textregions_dilated) contours_only_dilated = return_parent_contours(contours_only_dilated, hir_on_text_dilated) @@ -3723,21 +3709,20 @@ class Eynollah: img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12, int(x_min_main[j]):int(x_max_main[j])] = 1 co_text_all_org = contours_only_text_parent + contours_only_text_parent_h - if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: co_text_all = contours_only_dilated + contours_only_text_parent_h else: co_text_all = contours_only_text_parent + contours_only_text_parent_h else: co_text_all_org = contours_only_text_parent - if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: co_text_all = contours_only_dilated else: co_text_all = contours_only_text_parent if not len(co_text_all): return [], [] - print(len(co_text_all), "co_text_all") - print(len(co_text_all_org), "co_text_all_org") + labels_con = np.zeros((int(y_len /6.), int(x_len/6.), len(co_text_all)), dtype=bool) co_text_all = [(i/6).astype(int) for i in co_text_all] for i in range(len(co_text_all)): @@ -3805,7 +3790,7 @@ class Eynollah: ordered = [i[0] for i in ordered] - if len(contours_only_text_parent)>min_cont_size_to_be_dilated: + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: org_contours_indexes = [] for ind in range(len(ordered)): region_with_curr_order = ordered[ind] @@ -3823,215 +3808,6 @@ class Eynollah: else: region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] return ordered, region_ids - - - ####def return_start_and_end_of_common_text_of_textline_ocr(self, textline_image, ind_tot): - ####width = np.shape(textline_image)[1] - ####height = np.shape(textline_image)[0] - ####common_window = int(0.2*width) - - ####width1 = int ( width/2. - common_window ) - ####width2 = int ( width/2. + common_window ) - - ####img_sum = np.sum(textline_image[:,:,0], axis=0) - ####sum_smoothed = gaussian_filter1d(img_sum, 3) - - ####peaks_real, _ = find_peaks(sum_smoothed, height=0) - ####if len(peaks_real)>70: - - ####peaks_real = peaks_real[(peaks_realwidth1)] - - ####arg_sort = np.argsort(sum_smoothed[peaks_real]) - ####arg_sort4 =arg_sort[::-1][:4] - ####peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - ####argsort_sorted = np.argsort(peaks_sort_4) - - ####first_4_sorted = peaks_sort_4[argsort_sorted] - ####y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] - #####print(first_4_sorted,'first_4_sorted') - - ####arg_sortnew = np.argsort(y_4_sorted) - ####peaks_final =np.sort( first_4_sorted[arg_sortnew][2:] ) - - #####plt.figure(ind_tot) - #####plt.imshow(textline_image) - #####plt.plot([peaks_final[0], peaks_final[0]], [0, height-1]) - #####plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - #####plt.savefig('./'+str(ind_tot)+'.png') - - ####return peaks_final[0], peaks_final[1] - ####else: - ####pass - - ##def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image, ind_tot): - ##width = np.shape(textline_image)[1] - ##height = np.shape(textline_image)[0] - ##common_window = int(0.06*width) - - ##width1 = int ( width/2. - common_window ) - ##width2 = int ( width/2. + common_window ) - - ##img_sum = np.sum(textline_image[:,:,0], axis=0) - ##sum_smoothed = gaussian_filter1d(img_sum, 3) - - ##peaks_real, _ = find_peaks(sum_smoothed, height=0) - ##if len(peaks_real)>70: - ###print(len(peaks_real), 'len(peaks_real)') - - ##peaks_real = peaks_real[(peaks_realwidth1)] - - ##arg_max = np.argmax(sum_smoothed[peaks_real]) - ##peaks_final = peaks_real[arg_max] - - ###plt.figure(ind_tot) - ###plt.imshow(textline_image) - ###plt.plot([peaks_final, peaks_final], [0, height-1]) - ####plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - ###plt.savefig('./'+str(ind_tot)+'.png') - - ##return peaks_final - ##else: - ##return None - - ###def return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - ###self, peaks_real, sum_smoothed, start_split, end_split): - - ###peaks_real = peaks_real[(peaks_realstart_split)] - - ###arg_sort = np.argsort(sum_smoothed[peaks_real]) - ###arg_sort4 =arg_sort[::-1][:4] - ###peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - ###argsort_sorted = np.argsort(peaks_sort_4) - - ###first_4_sorted = peaks_sort_4[argsort_sorted] - ###y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] - ####print(first_4_sorted,'first_4_sorted') - - ###arg_sortnew = np.argsort(y_4_sorted) - ###peaks_final =np.sort( first_4_sorted[arg_sortnew][3:] ) - ###return peaks_final[0] - - ###def return_start_and_end_of_common_text_of_textline_ocr_new(self, textline_image, ind_tot): - ###width = np.shape(textline_image)[1] - ###height = np.shape(textline_image)[0] - ###common_window = int(0.15*width) - - ###width1 = int ( width/2. - common_window ) - ###width2 = int ( width/2. + common_window ) - ###mid = int(width/2.) - - ###img_sum = np.sum(textline_image[:,:,0], axis=0) - ###sum_smoothed = gaussian_filter1d(img_sum, 3) - - ###peaks_real, _ = find_peaks(sum_smoothed, height=0) - ###if len(peaks_real)>70: - ###peak_start = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - ###peaks_real, sum_smoothed, width1, mid+2) - ###peak_end = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - ###peaks_real, sum_smoothed, mid-2, width2) - - ####plt.figure(ind_tot) - ####plt.imshow(textline_image) - ####plt.plot([peak_start, peak_start], [0, height-1]) - ####plt.plot([peak_end, peak_end], [0, height-1]) - ####plt.savefig('./'+str(ind_tot)+'.png') - - ###return peak_start, peak_end - ###else: - ###pass - - ##def return_ocr_of_textline_without_common_section( - ##self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): - - ##if h2w_ratio > 0.05: - ##pixel_values = processor(textline_image, return_tensors="pt").pixel_values - ##generated_ids = model_ocr.generate(pixel_values.to(device)) - ##generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - ##else: - ###width = np.shape(textline_image)[1] - ###height = np.shape(textline_image)[0] - ###common_window = int(0.3*width) - ###width1 = int ( width/2. - common_window ) - ###width2 = int ( width/2. + common_window ) - - ##split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section( - ##textline_image, ind_tot) - ##if split_point: - ##image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) - ##image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) - - ###pixel_values1 = processor(image1, return_tensors="pt").pixel_values - ###pixel_values2 = processor(image2, return_tensors="pt").pixel_values - - ##pixel_values_merged = processor([image1,image2], return_tensors="pt").pixel_values - ##generated_ids_merged = model_ocr.generate(pixel_values_merged.to(device)) - ##generated_text_merged = processor.batch_decode(generated_ids_merged, skip_special_tokens=True) - - ###print(generated_text_merged,'generated_text_merged') - - ###generated_ids1 = model_ocr.generate(pixel_values1.to(device)) - ###generated_ids2 = model_ocr.generate(pixel_values2.to(device)) - - ###generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] - ###generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] - - ###generated_text = generated_text1 + ' ' + generated_text2 - ##generated_text = generated_text_merged[0] + ' ' + generated_text_merged[1] - - ###print(generated_text1,'generated_text1') - ###print(generated_text2, 'generated_text2') - ###print('########################################') - ##else: - ##pixel_values = processor(textline_image, return_tensors="pt").pixel_values - ##generated_ids = model_ocr.generate(pixel_values.to(device)) - ##generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - - ###print(generated_text,'generated_text') - ###print('########################################') - ##return generated_text - - ###def return_ocr_of_textline( - ###self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): - - ###if h2w_ratio > 0.05: - ###pixel_values = processor(textline_image, return_tensors="pt").pixel_values - ###generated_ids = model_ocr.generate(pixel_values.to(device)) - ###generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - ###else: - ####width = np.shape(textline_image)[1] - ####height = np.shape(textline_image)[0] - ####common_window = int(0.3*width) - ####width1 = int ( width/2. - common_window ) - ####width2 = int ( width/2. + common_window ) - - ###try: - ###width1, width2 = self.return_start_and_end_of_common_text_of_textline_ocr_new(textline_image, ind_tot) - - ###image1 = textline_image[:, :width2,:]# image.crop((0, 0, width2, height)) - ###image2 = textline_image[:, width1:,:]#image.crop((width1, 0, width, height)) - - ###pixel_values1 = processor(image1, return_tensors="pt").pixel_values - ###pixel_values2 = processor(image2, return_tensors="pt").pixel_values - - ###generated_ids1 = model_ocr.generate(pixel_values1.to(device)) - ###generated_ids2 = model_ocr.generate(pixel_values2.to(device)) - - ###generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] - ###generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] - ####print(generated_text1,'generated_text1') - ####print(generated_text2, 'generated_text2') - ####print('########################################') - - ###match = sq(None, generated_text1, generated_text2).find_longest_match( - ###0, len(generated_text1), 0, len(generated_text2)) - ###generated_text = generated_text1 + generated_text2[match.b+match.size:] - ###except: - ###pixel_values = processor(textline_image, return_tensors="pt").pixel_values - ###generated_ids = model_ocr.generate(pixel_values.to(device)) - ###generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - - ###return generated_text - def return_list_of_contours_with_desired_order(self, ls_cons, sorted_indexes): return [ls_cons[sorted_indexes[index]] for index in range(len(sorted_indexes))] diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py new file mode 100644 index 0000000..71445f7 --- /dev/null +++ b/src/eynollah/image_enhancer.py @@ -0,0 +1,756 @@ +""" +Image enhancer. The output can be written as same scale of input or in new predicted scale. +""" + +from logging import Logger +from difflib import SequenceMatcher as sq +from PIL import Image, ImageDraw, ImageFont +import math +import os +import sys +import time +from typing import Optional +import atexit +import warnings +from functools import partial +from pathlib import Path +from multiprocessing import cpu_count +import gc +import copy +from loky import ProcessPoolExecutor +import xml.etree.ElementTree as ET +import cv2 +import numpy as np +from ocrd import OcrdPage +from ocrd_utils import getLogger, tf_disable_interactive_logs +import statistics +from tensorflow.keras.models import load_model +from .utils.resize import resize_image +from .utils import ( + crop_image_inside_box +) + +DPI_THRESHOLD = 298 +KERNEL = np.ones((5, 5), np.uint8) + + +class Enhancer: + def __init__( + self, + dir_models : str, + dir_out : Optional[str] = None, + num_col_upper : Optional[int] = None, + num_col_lower : Optional[int] = None, + logger : Optional[Logger] = None, + ): + self.dir_out = dir_out + self.input_binary = False + self.light_version = False + if num_col_upper: + self.num_col_upper = int(num_col_upper) + else: + self.num_col_upper = num_col_upper + if num_col_lower: + self.num_col_lower = int(num_col_lower) + else: + self.num_col_lower = num_col_lower + + self.logger = logger if logger else getLogger('enhancement') + # for parallelization of CPU-intensive tasks: + self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) + atexit.register(self.executor.shutdown) + self.dir_models = dir_models + self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" + self.model_dir_of_col_classifier = dir_models + "/eynollah-column-classifier_20210425" + self.model_page_dir = dir_models + "/eynollah-page-extraction_20210425" + + try: + for device in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(device, True) + except: + self.logger.warning("no GPU device available") + + self.model_page = self.our_load_model(self.model_page_dir) + self.model_classifier = self.our_load_model(self.model_dir_of_col_classifier) + self.model_enhancement = self.our_load_model(self.model_dir_of_enhancement) + + def cache_images(self, image_filename=None, image_pil=None, dpi=None): + ret = {} + t_c0 = time.time() + if image_filename: + ret['img'] = cv2.imread(image_filename) + if self.light_version: + self.dpi = 100 + else: + self.dpi = 0#check_dpi(image_filename) + else: + ret['img'] = pil2cv(image_pil) + if self.light_version: + self.dpi = 100 + else: + self.dpi = 0#check_dpi(image_pil) + ret['img_grayscale'] = cv2.cvtColor(ret['img'], cv2.COLOR_BGR2GRAY) + for prefix in ('', '_grayscale'): + ret[f'img{prefix}_uint8'] = ret[f'img{prefix}'].astype(np.uint8) + self._imgs = ret + if dpi is not None: + self.dpi = dpi + + def reset_file_name_dir(self, image_filename): + t_c = time.time() + self.cache_images(image_filename=image_filename) + self.output_filename = os.path.join(self.dir_out, Path(image_filename).stem +'.png') + + def imread(self, grayscale=False, uint8=True): + key = 'img' + if grayscale: + key += '_grayscale' + if uint8: + key += '_uint8' + return self._imgs[key].copy() + + def isNaN(self, num): + return num != num + + @staticmethod + def our_load_model(model_file): + if model_file.endswith('.h5') and Path(model_file[:-3]).exists(): + # prefer SavedModel over HDF5 format if it exists + model_file = model_file[:-3] + try: + model = load_model(model_file, compile=False) + except: + model = load_model(model_file, compile=False, custom_objects={ + "PatchEncoder": PatchEncoder, "Patches": Patches}) + return model + + def predict_enhancement(self, img): + self.logger.debug("enter predict_enhancement") + + img_height_model = self.model_enhancement.layers[-1].output_shape[1] + img_width_model = self.model_enhancement.layers[-1].output_shape[2] + if img.shape[0] < img_height_model: + img = cv2.resize(img, (img.shape[1], img_width_model), interpolation=cv2.INTER_NEAREST) + if img.shape[1] < img_width_model: + img = cv2.resize(img, (img_height_model, img.shape[0]), interpolation=cv2.INTER_NEAREST) + margin = int(0.1 * img_width_model) + width_mid = img_width_model - 2 * margin + height_mid = img_height_model - 2 * margin + img = img / 255. + img_h = img.shape[0] + img_w = img.shape[1] + + prediction_true = np.zeros((img_h, img_w, 3)) + nxf = img_w / float(width_mid) + nyf = img_h / float(height_mid) + nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) + nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + + for i in range(nxf): + for j in range(nyf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + else: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + if j == 0: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + else: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - img_width_model + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - img_height_model + + img_patch = img[np.newaxis, index_y_d:index_y_u, index_x_d:index_x_u, :] + label_p_pred = self.model_enhancement.predict(img_patch, verbose=0) + seg = label_p_pred[0, :, :, :] * 255 + + if i == 0 and j == 0: + prediction_true[index_y_d + 0:index_y_u - margin, + index_x_d + 0:index_x_u - margin] = \ + seg[0:-margin or None, + 0:-margin or None] + elif i == nxf - 1 and j == nyf - 1: + prediction_true[index_y_d + margin:index_y_u - 0, + index_x_d + margin:index_x_u - 0] = \ + seg[margin:, + margin:] + elif i == 0 and j == nyf - 1: + prediction_true[index_y_d + margin:index_y_u - 0, + index_x_d + 0:index_x_u - margin] = \ + seg[margin:, + 0:-margin or None] + elif i == nxf - 1 and j == 0: + prediction_true[index_y_d + 0:index_y_u - margin, + index_x_d + margin:index_x_u - 0] = \ + seg[0:-margin or None, + margin:] + elif i == 0 and j != 0 and j != nyf - 1: + prediction_true[index_y_d + margin:index_y_u - margin, + index_x_d + 0:index_x_u - margin] = \ + seg[margin:-margin or None, + 0:-margin or None] + elif i == nxf - 1 and j != 0 and j != nyf - 1: + prediction_true[index_y_d + margin:index_y_u - margin, + index_x_d + margin:index_x_u - 0] = \ + seg[margin:-margin or None, + margin:] + elif i != 0 and i != nxf - 1 and j == 0: + prediction_true[index_y_d + 0:index_y_u - margin, + index_x_d + margin:index_x_u - margin] = \ + seg[0:-margin or None, + margin:-margin or None] + elif i != 0 and i != nxf - 1 and j == nyf - 1: + prediction_true[index_y_d + margin:index_y_u - 0, + index_x_d + margin:index_x_u - margin] = \ + seg[margin:, + margin:-margin or None] + else: + prediction_true[index_y_d + margin:index_y_u - margin, + index_x_d + margin:index_x_u - margin] = \ + seg[margin:-margin or None, + margin:-margin or None] + + prediction_true = prediction_true.astype(int) + return prediction_true + + def calculate_width_height_by_columns(self, img, num_col, width_early, label_p_pred): + self.logger.debug("enter calculate_width_height_by_columns") + if num_col == 1 and width_early < 1100: + img_w_new = 2000 + elif num_col == 1 and width_early >= 2500: + img_w_new = 2000 + elif num_col == 1 and width_early >= 1100 and width_early < 2500: + img_w_new = width_early + elif num_col == 2 and width_early < 2000: + img_w_new = 2400 + elif num_col == 2 and width_early >= 3500: + img_w_new = 2400 + elif num_col == 2 and width_early >= 2000 and width_early < 3500: + img_w_new = width_early + elif num_col == 3 and width_early < 2000: + img_w_new = 3000 + elif num_col == 3 and width_early >= 4000: + img_w_new = 3000 + elif num_col == 3 and width_early >= 2000 and width_early < 4000: + img_w_new = width_early + elif num_col == 4 and width_early < 2500: + img_w_new = 4000 + elif num_col == 4 and width_early >= 5000: + img_w_new = 4000 + elif num_col == 4 and width_early >= 2500 and width_early < 5000: + img_w_new = width_early + elif num_col == 5 and width_early < 3700: + img_w_new = 5000 + elif num_col == 5 and width_early >= 7000: + img_w_new = 5000 + elif num_col == 5 and width_early >= 3700 and width_early < 7000: + img_w_new = width_early + elif num_col == 6 and width_early < 4500: + img_w_new = 6500 # 5400 + else: + img_w_new = width_early + img_h_new = img_w_new * img.shape[0] // img.shape[1] + + if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: + img_new = np.copy(img) + num_column_is_classified = False + #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: + elif img_h_new >= 8000: + img_new = np.copy(img) + num_column_is_classified = False + else: + img_new = resize_image(img, img_h_new, img_w_new) + num_column_is_classified = True + + return img_new, num_column_is_classified + + def early_page_for_num_of_column_classification(self,img_bin): + self.logger.debug("enter early_page_for_num_of_column_classification") + if self.input_binary: + img = np.copy(img_bin).astype(np.uint8) + else: + img = self.imread() + img = cv2.GaussianBlur(img, (5, 5), 0) + img_page_prediction = self.do_prediction(False, img, self.model_page) + + imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + thresh = cv2.dilate(thresh, KERNEL, iterations=3) + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + if len(contours)>0: + cnt_size = np.array([cv2.contourArea(contours[j]) + for j in range(len(contours))]) + cnt = contours[np.argmax(cnt_size)] + box = cv2.boundingRect(cnt) + else: + box = [0, 0, img.shape[1], img.shape[0]] + cropped_page, page_coord = crop_image_inside_box(box, img) + + self.logger.debug("exit early_page_for_num_of_column_classification") + return cropped_page, page_coord + + def calculate_width_height_by_columns_1_2(self, img, num_col, width_early, label_p_pred): + self.logger.debug("enter calculate_width_height_by_columns") + if num_col == 1: + img_w_new = 1000 + else: + img_w_new = 1300 + img_h_new = img_w_new * img.shape[0] // img.shape[1] + + if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: + img_new = np.copy(img) + num_column_is_classified = False + #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: + elif img_h_new >= 8000: + img_new = np.copy(img) + num_column_is_classified = False + else: + img_new = resize_image(img, img_h_new, img_w_new) + num_column_is_classified = True + + return img_new, num_column_is_classified + + def resize_and_enhance_image_with_column_classifier(self, light_version): + self.logger.debug("enter resize_and_enhance_image_with_column_classifier") + dpi = 0#self.dpi + self.logger.info("Detected %s DPI", dpi) + if self.input_binary: + img = self.imread() + prediction_bin = self.do_prediction(True, img, self.model_bin, n_batch_inference=5) + prediction_bin = 255 * (prediction_bin[:,:,0]==0) + prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2).astype(np.uint8) + img= np.copy(prediction_bin) + img_bin = prediction_bin + else: + img = self.imread() + self.h_org, self.w_org = img.shape[:2] + img_bin = None + + width_early = img.shape[1] + t1 = time.time() + _, page_coord = self.early_page_for_num_of_column_classification(img_bin) + + self.image_page_org_size = img[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3], :] + self.page_coord = page_coord + + if self.num_col_upper and not self.num_col_lower: + num_col = self.num_col_upper + label_p_pred = [np.ones(6)] + elif self.num_col_lower and not self.num_col_upper: + num_col = self.num_col_lower + label_p_pred = [np.ones(6)] + elif not self.num_col_upper and not self.num_col_lower: + if self.input_binary: + img_in = np.copy(img) + img_in = img_in / 255.0 + img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = img_in.reshape(1, 448, 448, 3) + else: + img_1ch = self.imread(grayscale=True) + width_early = img_1ch.shape[1] + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + img_1ch = img_1ch / 255.0 + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = self.model_classifier.predict(img_in, verbose=0) + num_col = np.argmax(label_p_pred[0]) + 1 + elif (self.num_col_upper and self.num_col_lower) and (self.num_col_upper!=self.num_col_lower): + if self.input_binary: + img_in = np.copy(img) + img_in = img_in / 255.0 + img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = img_in.reshape(1, 448, 448, 3) + else: + img_1ch = self.imread(grayscale=True) + width_early = img_1ch.shape[1] + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + img_1ch = img_1ch / 255.0 + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = self.model_classifier.predict(img_in, verbose=0) + num_col = np.argmax(label_p_pred[0]) + 1 + + if num_col > self.num_col_upper: + num_col = self.num_col_upper + label_p_pred = [np.ones(6)] + if num_col < self.num_col_lower: + num_col = self.num_col_lower + label_p_pred = [np.ones(6)] + else: + num_col = self.num_col_upper + label_p_pred = [np.ones(6)] + + self.logger.info("Found %d columns (%s)", num_col, np.around(label_p_pred, decimals=5)) + + if dpi < DPI_THRESHOLD: + if light_version and num_col in (1,2): + img_new, num_column_is_classified = self.calculate_width_height_by_columns_1_2( + img, num_col, width_early, label_p_pred) + else: + img_new, num_column_is_classified = self.calculate_width_height_by_columns( + img, num_col, width_early, label_p_pred) + if light_version: + image_res = np.copy(img_new) + else: + image_res = self.predict_enhancement(img_new) + is_image_enhanced = True + + else: + num_column_is_classified = True + image_res = np.copy(img) + is_image_enhanced = False + + self.logger.debug("exit resize_and_enhance_image_with_column_classifier") + return is_image_enhanced, img, image_res, num_col, num_column_is_classified, img_bin + def do_prediction( + self, patches, img, model, + n_batch_inference=1, marginal_of_patch_percent=0.1, + thresholding_for_some_classes_in_light_version=False, + thresholding_for_artificial_class_in_light_version=False, thresholding_for_fl_light_version=False, threshold_art_class_textline=0.1): + + self.logger.debug("enter do_prediction") + img_height_model = model.layers[-1].output_shape[1] + img_width_model = model.layers[-1].output_shape[2] + + if not patches: + img_h_page = img.shape[0] + img_w_page = img.shape[1] + img = img / float(255.0) + img = resize_image(img, img_height_model, img_width_model) + + label_p_pred = model.predict(img[np.newaxis], verbose=0) + seg = np.argmax(label_p_pred, axis=3)[0] + + if thresholding_for_artificial_class_in_light_version: + seg_art = label_p_pred[0,:,:,2] + + seg_art[seg_art0] =1 + + skeleton_art = skeletonize(seg_art) + skeleton_art = skeleton_art*1 + + seg[skeleton_art==1]=2 + + if thresholding_for_fl_light_version: + seg_header = label_p_pred[0,:,:,2] + + seg_header[seg_header<0.2] = 0 + seg_header[seg_header>0] =1 + + seg[seg_header==1]=2 + + seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) + prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) + return prediction_true + + if img.shape[0] < img_height_model: + img = resize_image(img, img_height_model, img.shape[1]) + if img.shape[1] < img_width_model: + img = resize_image(img, img.shape[0], img_width_model) + + self.logger.debug("Patch size: %sx%s", img_height_model, img_width_model) + margin = int(marginal_of_patch_percent * img_height_model) + width_mid = img_width_model - 2 * margin + height_mid = img_height_model - 2 * margin + img = img / 255. + #img = img.astype(np.float16) + img_h = img.shape[0] + img_w = img.shape[1] + prediction_true = np.zeros((img_h, img_w, 3)) + mask_true = np.zeros((img_h, img_w)) + nxf = img_w / float(width_mid) + nyf = img_h / float(height_mid) + nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) + nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + + list_i_s = [] + list_j_s = [] + list_x_u = [] + list_x_d = [] + list_y_u = [] + list_y_d = [] + + batch_indexer = 0 + img_patch = np.zeros((n_batch_inference, img_height_model, img_width_model, 3)) + for i in range(nxf): + for j in range(nyf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + else: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + if j == 0: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + else: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - img_width_model + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - img_height_model + + list_i_s.append(i) + list_j_s.append(j) + list_x_u.append(index_x_u) + list_x_d.append(index_x_d) + list_y_d.append(index_y_d) + list_y_u.append(index_y_u) + + img_patch[batch_indexer,:,:,:] = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + batch_indexer += 1 + + if (batch_indexer == n_batch_inference or + # last batch + i == nxf - 1 and j == nyf - 1): + self.logger.debug("predicting patches on %s", str(img_patch.shape)) + label_p_pred = model.predict(img_patch, verbose=0) + seg = np.argmax(label_p_pred, axis=3) + + if thresholding_for_some_classes_in_light_version: + seg_not_base = label_p_pred[:,:,:,4] + seg_not_base[seg_not_base>0.03] =1 + seg_not_base[seg_not_base<1] =0 + + seg_line = label_p_pred[:,:,:,3] + seg_line[seg_line>0.1] =1 + seg_line[seg_line<1] =0 + + seg_background = label_p_pred[:,:,:,0] + seg_background[seg_background>0.25] =1 + seg_background[seg_background<1] =0 + + seg[seg_not_base==1]=4 + seg[seg_background==1]=0 + seg[(seg_line==1) & (seg==0)]=3 + if thresholding_for_artificial_class_in_light_version: + seg_art = label_p_pred[:,:,:,2] + + seg_art[seg_art0] =1 + + ##seg[seg_art==1]=2 + + indexer_inside_batch = 0 + for i_batch, j_batch in zip(list_i_s, list_j_s): + seg_in = seg[indexer_inside_batch] + + if thresholding_for_artificial_class_in_light_version: + seg_in_art = seg_art[indexer_inside_batch] + + index_y_u_in = list_y_u[indexer_inside_batch] + index_y_d_in = list_y_d[indexer_inside_batch] + + index_x_u_in = list_x_u[indexer_inside_batch] + index_x_d_in = list_x_d[indexer_inside_batch] + + if i_batch == 0 and j_batch == 0: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin] = \ + seg_in[0:-margin or None, + 0:-margin or None, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[0:-margin or None, + 0:-margin or None] + + elif i_batch == nxf - 1 and j_batch == nyf - 1: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - 0] = \ + seg_in[margin:, + margin:, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[margin:, + margin:] + + elif i_batch == 0 and j_batch == nyf - 1: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + 0:index_x_u_in - margin] = \ + seg_in[margin:, + 0:-margin or None, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[margin:, + 0:-margin or None] + + elif i_batch == nxf - 1 and j_batch == 0: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0] = \ + seg_in[0:-margin or None, + margin:, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[0:-margin or None, + margin:] + + elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin] = \ + seg_in[margin:-margin or None, + 0:-margin or None, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + 0:index_x_u_in - margin, 1] = \ + seg_in_art[margin:-margin or None, + 0:-margin or None] + + elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0] = \ + seg_in[margin:-margin or None, + margin:, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - 0, 1] = \ + seg_in_art[margin:-margin or None, + margin:] + + elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin] = \ + seg_in[0:-margin or None, + margin:-margin or None, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + 0:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[0:-margin or None, + margin:-margin or None] + + elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - margin] = \ + seg_in[margin:, + margin:-margin or None, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - 0, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[margin:, + margin:-margin or None] + + else: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin] = \ + seg_in[margin:-margin or None, + margin:-margin or None, + np.newaxis] + if thresholding_for_artificial_class_in_light_version: + prediction_true[index_y_d_in + margin:index_y_u_in - margin, + index_x_d_in + margin:index_x_u_in - margin, 1] = \ + seg_in_art[margin:-margin or None, + margin:-margin or None] + indexer_inside_batch += 1 + + + list_i_s = [] + list_j_s = [] + list_x_u = [] + list_x_d = [] + list_y_u = [] + list_y_d = [] + + batch_indexer = 0 + img_patch[:] = 0 + + prediction_true = prediction_true.astype(np.uint8) + + if thresholding_for_artificial_class_in_light_version: + kernel_min = np.ones((3, 3), np.uint8) + prediction_true[:,:,0][prediction_true[:,:,0]==2] = 0 + + skeleton_art = skeletonize(prediction_true[:,:,1]) + skeleton_art = skeleton_art*1 + + skeleton_art = skeleton_art.astype('uint8') + + skeleton_art = cv2.dilate(skeleton_art, kernel_min, iterations=1) + + prediction_true[:,:,0][skeleton_art==1]=2 + #del model + gc.collect() + return prediction_true + + def run_enhancement(self, light_version): + t_in = time.time() + self.logger.info("Resizing and enhancing image...") + is_image_enhanced, img_org, img_res, num_col_classifier, num_column_is_classified, img_bin = \ + self.resize_and_enhance_image_with_column_classifier(light_version) + + self.logger.info("Image was %senhanced.", '' if is_image_enhanced else 'not ') + return img_res, is_image_enhanced, num_col_classifier, num_column_is_classified + + + def run_single(self): + t0 = time.time() + img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(light_version=False) + + return img_res + + + def run(self, image_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): + """ + Get image and scales, then extract the page of scanned image + """ + self.logger.debug("enter run") + t0_tot = time.time() + + if dir_in: + self.ls_imgs = os.listdir(dir_in) + elif image_filename: + self.ls_imgs = [image_filename] + else: + raise ValueError("run requires either a single image filename or a directory") + + for img_filename in self.ls_imgs: + self.logger.info(img_filename) + t0 = time.time() + + self.reset_file_name_dir(os.path.join(dir_in or "", img_filename)) + #print("text region early -11 in %.1fs", time.time() - t0) + + if os.path.exists(self.output_filename): + if overwrite: + self.logger.warning("will overwrite existing output file '%s'", self.output_filename) + else: + self.logger.warning("will skip input for existing output file '%s'", self.output_filename) + continue + + image_enhanced = self.run_single() + img_enhanced_org_scale = resize_image(image_enhanced, self.h_org, self.w_org) + + cv2.imwrite(self.output_filename, img_enhanced_org_scale) + From 9342b76038fb274e1f4f8a7e2d31cb1ee3e1e296 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 1 Jun 2025 22:10:13 +0200 Subject: [PATCH 154/374] saving enhanced image in org or scaled resolution --- src/eynollah/cli.py | 9 ++++++++- src/eynollah/eynollah.py | 5 ++--- src/eynollah/image_enhancer.py | 7 +++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 840bc4b..9398c47 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -116,6 +116,12 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) "-ncl", help="upper limit of columns in document image", ) +@click.option( + "--save_org_scale/--no_save_org_scale", + "-sos/-nosos", + is_flag=True, + help="if this parameter set to true, this tool will save the enhanced image in org scale.", +) @click.option( "--log_level", "-l", @@ -123,7 +129,7 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) help="Override log level globally to this", ) -def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_lower, log_level): +def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_lower, save_org_scale, log_level): initLogging() if log_level: getLogger('enhancement').setLevel(getLevelName(log_level)) @@ -134,6 +140,7 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low dir_out=out, num_col_upper=num_col_upper, num_col_lower=num_col_lower, + save_org_scale=save_org_scale, ) if dir_in: enhancer_object.run(dir_in=dir_in, overwrite=overwrite) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index cf540d3..9c834e2 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5434,10 +5434,9 @@ class Eynollah_ocr: img_crop = img_poly_on_img[y:y+h, x:x+w, :] - #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') if not self.do_not_mask_with_textline_contour: - if angle_degrees > 15: + if angle_degrees > 3: better_des_slope = get_orientation_moments(textline_coords) img_crop = rotate_image_with_padding(img_crop, better_des_slope ) @@ -5484,7 +5483,7 @@ class Eynollah_ocr: if not self.export_textline_images_and_text: - if w_scaled < 530:#640:#1.5*image_width: + if w_scaled < 640:#1.5*image_width: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) if angle_degrees > 15: diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index 71445f7..c89f532 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -41,11 +41,13 @@ class Enhancer: dir_out : Optional[str] = None, num_col_upper : Optional[int] = None, num_col_lower : Optional[int] = None, + save_org_scale : bool = False, logger : Optional[Logger] = None, ): self.dir_out = dir_out self.input_binary = False self.light_version = False + self.save_org_scale = save_org_scale if num_col_upper: self.num_col_upper = int(num_col_upper) else: @@ -750,7 +752,8 @@ class Enhancer: continue image_enhanced = self.run_single() - img_enhanced_org_scale = resize_image(image_enhanced, self.h_org, self.w_org) + if self.save_org_scale: + image_enhanced = resize_image(image_enhanced, self.h_org, self.w_org) - cv2.imwrite(self.output_filename, img_enhanced_org_scale) + cv2.imwrite(self.output_filename, image_enhanced) From e26c4ab9b4071df22445fc6b45d91db826ce7917 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 1 Jun 2025 22:44:50 +0200 Subject: [PATCH 155/374] image enhancer updated --- src/eynollah/image_enhancer.py | 40 +++++++--------------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index c89f532..983712d 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -225,47 +225,23 @@ class Enhancer: def calculate_width_height_by_columns(self, img, num_col, width_early, label_p_pred): self.logger.debug("enter calculate_width_height_by_columns") - if num_col == 1 and width_early < 1100: + if num_col == 1: img_w_new = 2000 - elif num_col == 1 and width_early >= 2500: - img_w_new = 2000 - elif num_col == 1 and width_early >= 1100 and width_early < 2500: - img_w_new = width_early - elif num_col == 2 and width_early < 2000: + elif num_col == 2: img_w_new = 2400 - elif num_col == 2 and width_early >= 3500: - img_w_new = 2400 - elif num_col == 2 and width_early >= 2000 and width_early < 3500: - img_w_new = width_early - elif num_col == 3 and width_early < 2000: + elif num_col == 3: img_w_new = 3000 - elif num_col == 3 and width_early >= 4000: - img_w_new = 3000 - elif num_col == 3 and width_early >= 2000 and width_early < 4000: - img_w_new = width_early - elif num_col == 4 and width_early < 2500: + elif num_col == 4: img_w_new = 4000 - elif num_col == 4 and width_early >= 5000: - img_w_new = 4000 - elif num_col == 4 and width_early >= 2500 and width_early < 5000: - img_w_new = width_early - elif num_col == 5 and width_early < 3700: + elif num_col == 5: img_w_new = 5000 - elif num_col == 5 and width_early >= 7000: - img_w_new = 5000 - elif num_col == 5 and width_early >= 3700 and width_early < 7000: - img_w_new = width_early - elif num_col == 6 and width_early < 4500: - img_w_new = 6500 # 5400 + elif num_col == 6: + img_w_new = 6500 else: img_w_new = width_early img_h_new = img_w_new * img.shape[0] // img.shape[1] - if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: - img_new = np.copy(img) - num_column_is_classified = False - #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: - elif img_h_new >= 8000: + if img_h_new >= 8000: img_new = np.copy(img) num_column_is_classified = False else: From f79af201abf14b2fe6ec51b066daf7aac7a929ff Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 2 Jun 2025 18:21:33 +0200 Subject: [PATCH 156/374] Fix: Resolved OCR bug when text region type is undefined --- src/eynollah/eynollah.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9c834e2..fc60f2e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5399,7 +5399,10 @@ class Eynollah_ocr: indexer_text_region = 0 indexer_textlines = 0 for nn in root1.iter(region_tags): - type_textregion = nn.attrib['type'] + try: + type_textregion = nn.attrib['type'] + except: + type_textregion = 'paragraph' for child_textregion in nn: if child_textregion.tag.endswith("TextLine"): for child_textlines in child_textregion: @@ -5467,6 +5470,7 @@ class Eynollah_ocr: else: + better_des_slope = 0 img_crop[mask_poly==0] = 255 if self.prediction_with_both_of_rgb_and_bin: img_crop_bin[mask_poly==0] = 255 @@ -5486,7 +5490,7 @@ class Eynollah_ocr: if w_scaled < 640:#1.5*image_width: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) - if angle_degrees > 15: + if abs(better_des_slope) > 45: cropped_lines_ver_index.append(1) else: cropped_lines_ver_index.append(0) @@ -5505,7 +5509,7 @@ class Eynollah_ocr: cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(1) - if angle_degrees > 15: + if abs(better_des_slope) > 45: cropped_lines_ver_index.append(1) else: cropped_lines_ver_index.append(0) @@ -5515,7 +5519,7 @@ class Eynollah_ocr: cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(-1) - if angle_degrees > 15: + if abs(better_des_slope) > 45: cropped_lines_ver_index.append(1) else: cropped_lines_ver_index.append(0) @@ -5531,7 +5535,7 @@ class Eynollah_ocr: cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) - if angle_degrees > 15: + if abs(better_des_slope) > 45: cropped_lines_ver_index.append(1) else: cropped_lines_ver_index.append(0) From eb91000490282e2ea0d6058032f69f29da7783b6 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 2 Jun 2025 18:23:34 +0200 Subject: [PATCH 157/374] layout visualization updated --- train/generate_gt_for_training.py | 4 ++-- train/gt_gen_utils.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 9b7f02b..8ca5cd3 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -418,7 +418,7 @@ def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): help="directory of images where textline segmentation will be overlayed", ) def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): - assert xml_file and dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" + assert xml_file or dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" if dir_xml: xml_files_ind = os.listdir(dir_xml) else: @@ -442,7 +442,7 @@ def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): co_text, co_graphic, co_sep, co_img, co_table, co_noise, y_len, x_len = get_layout_contours_for_visualization(xml_file) - added_image = visualize_image_from_contours_layout(co_text['paragraph'], co_text['header'], co_text['drop-capital'], co_sep, co_img, co_text['marginalia'], img) + added_image = visualize_image_from_contours_layout(co_text['paragraph'], co_text['header']+co_text['heading'], co_text['drop-capital'], co_sep, co_img, co_text['marginalia'], img) cv2.imwrite(os.path.join(dir_out, f_name+'.png'), added_image) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index a734020..0ac15a2 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -306,6 +306,7 @@ def get_layout_contours_for_visualization(xml_file): co_noise=[] types_text = [] + types_graphic = [] for tag in region_tags: if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): @@ -325,6 +326,9 @@ def get_layout_contours_for_visualization(xml_file): if len(types_text_without_paragraph) == 0: if "type" in nn.attrib: c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + elif len(types_text_without_paragraph) >= 1: if "type" in nn.attrib: if nn.attrib['type'] in types_text_without_paragraph: @@ -332,10 +336,15 @@ def get_layout_contours_for_visualization(xml_file): else: c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: if "type" in nn.attrib: if nn.attrib['type'] in all_defined_textregion_types: c_t_in[nn.attrib['type']].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + c_t_in['paragraph'].append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) break else: From 9b4e78c55ce4fc4c121a9e6afae4ebcf79f42435 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 11 Jun 2025 18:57:08 +0200 Subject: [PATCH 158/374] Fixed duplicate textline_light assignments (true and false) in the OCR-D framework for the Eynollah light version, which caused rectangles to be used instead of contours for textlines --- src/eynollah/ocrd-tool.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index e972ec8..ce15206 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -38,7 +38,7 @@ "textline_light": { "type": "boolean", "default": true, - "description": "Light version need textline light" + "description": "Light version need textline light. If this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline with a faster method." }, "tables": { "type": "boolean", @@ -65,11 +65,6 @@ "default": false, "description": "if this parameter set to true, this tool would check that input image need resizing and enhancement or not." }, - "textline_light": { - "type": "boolean", - "default": false, - "description": "if this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline with a faster method." - }, "right_to_left": { "type": "boolean", "default": false, From 32889ef1e01dd24f0b7d5dfe0ad2a6e12a910aeb Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 12 Jun 2025 13:57:41 +0200 Subject: [PATCH 159/374] adapt binarization CLI according to #156 --- src/eynollah/cli.py | 19 ++++++++----------- src/eynollah/sbb_binarize.py | 10 +++++----- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index c189aca..42f9bca 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -48,8 +48,7 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i @main.command() @click.option('--patches/--no-patches', default=True, help='by enabling this parameter you let the model to see the image in patches.') @click.option('--model_dir', '-m', type=click.Path(exists=True, file_okay=False), required=True, help='directory containing models for prediction') -@click.argument('input_image', required=False) -@click.argument('output_image', required=False) +@click.option("--input-image", "-i", help="input image", type=click.Path(exists=True, dir_okay=False)) @click.option( "--dir_in", "-di", @@ -57,16 +56,14 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i type=click.Path(exists=True, file_okay=False), ) @click.option( - "--dir_out", - "-do", - help="directory for output images", - type=click.Path(exists=True, file_okay=False), + "--output", + "-o", + help="output image (if using -i) or output image directory (if using -di)", + type=click.Path(file_okay=True, dir_okay=True), ) -def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out): - assert (dir_out is None) == (dir_in is None), "Options -di and -do are mutually dependent" - assert (input_image is None) == (output_image is None), "INPUT_IMAGE and OUTPUT_IMAGE are mutually dependent" - assert (dir_in is None) != (input_image is None), "Specify either -di and -do options, or INPUT_IMAGE and OUTPUT_IMAGE" - SbbBinarizer(model_dir).run(image_path=input_image, use_patches=patches, save=output_image, dir_in=dir_in, dir_out=dir_out) +def binarization(patches, model_dir, input_image, dir_in, output): + assert (dir_in is None) != (input_image is None), "Specify either -di and or -i not both" + SbbBinarizer(model_dir).run(image_path=input_image, use_patches=patches, output=output, dir_in=dir_in) diff --git a/src/eynollah/sbb_binarize.py b/src/eynollah/sbb_binarize.py index f43b6ba..2d5035f 100644 --- a/src/eynollah/sbb_binarize.py +++ b/src/eynollah/sbb_binarize.py @@ -314,8 +314,8 @@ class SbbBinarizer: prediction_true = prediction_true.astype(np.uint8) return prediction_true[:,:,0] - def run(self, image=None, image_path=None, save=None, use_patches=False, dir_in=None, dir_out=None): - print(dir_in,'dir_in') + def run(self, image=None, image_path=None, output=None, use_patches=False, dir_in=None): + # print(dir_in,'dir_in') if not dir_in: if (image is not None and image_path is not None) or \ (image is None and image_path is None): @@ -343,8 +343,8 @@ class SbbBinarizer: kernel = np.ones((5, 5), np.uint8) img_last[:, :][img_last[:, :] > 0] = 255 img_last = (img_last[:, :] == 0) * 255 - if save: - cv2.imwrite(save, img_last) + if output: + cv2.imwrite(output, img_last) return img_last else: ls_imgs = os.listdir(dir_in) @@ -374,4 +374,4 @@ class SbbBinarizer: img_last[:, :][img_last[:, :] > 0] = 255 img_last = (img_last[:, :] == 0) * 255 - cv2.imwrite(os.path.join(dir_out,image_stem+'.png'), img_last) + cv2.imwrite(os.path.join(output, image_stem + '.png'), img_last) From c194a20c9c55bedb16ed859343f48a6b3645eadc Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 11 Jun 2025 18:57:08 +0200 Subject: [PATCH 160/374] Fixed duplicate textline_light assignments (true and false) in the OCR-D framework for the Eynollah light version, which caused rectangles to be used instead of contours for textlines --- src/eynollah/ocrd-tool.json | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index e972ec8..ce15206 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -38,7 +38,7 @@ "textline_light": { "type": "boolean", "default": true, - "description": "Light version need textline light" + "description": "Light version need textline light. If this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline with a faster method." }, "tables": { "type": "boolean", @@ -65,11 +65,6 @@ "default": false, "description": "if this parameter set to true, this tool would check that input image need resizing and enhancement or not." }, - "textline_light": { - "type": "boolean", - "default": false, - "description": "if this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline with a faster method." - }, "right_to_left": { "type": "boolean", "default": false, From b7b218ff11660061fb0f606b871ebe3c9f831184 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 12 Jun 2025 15:30:17 +0200 Subject: [PATCH 161/374] OCR-D processor: same behavior as standalone wrt light_version/textline_light --- src/eynollah/processor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/eynollah/processor.py b/src/eynollah/processor.py index 8f99489..a53fede 100644 --- a/src/eynollah/processor.py +++ b/src/eynollah/processor.py @@ -14,9 +14,10 @@ class EynollahProcessor(Processor): return 'ocrd-eynollah-segment' def setup(self) -> None: - if self.parameter['textline_light'] and not self.parameter['light_version']: - raise ValueError("Error: You set parameter 'textline_light' to enable light textline detection, " - "but parameter 'light_version' is not enabled") + assert self.parameter + if self.parameter['textline_light'] != self.parameter['light_version']: + raise ValueError("Error: You must set or unset both parameter 'textline_light' (to enable light textline detection), " + "and parameter 'light_version' (faster+simpler method for main region detection and deskewing)") self.eynollah = Eynollah( self.resolve_resource(self.parameter['models']), logger=self.logger, From f5a1d1a255a080469ba4624d7912b6e5e4cc7647 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 25 Jun 2025 18:24:16 +0200 Subject: [PATCH 162/374] docker file to train model with desired cuda and cudnn --- train/Dockerfile | 29 ++++++++++++++++++ train/config_params_docker.json | 54 +++++++++++++++++++++++++++++++++ train/train.py | 2 +- 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 train/Dockerfile create mode 100644 train/config_params_docker.json diff --git a/train/Dockerfile b/train/Dockerfile new file mode 100644 index 0000000..2456ea4 --- /dev/null +++ b/train/Dockerfile @@ -0,0 +1,29 @@ +# Use NVIDIA base image +FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 + +# Set the working directory +WORKDIR /app + + +# Set environment variable for GitPython +ENV GIT_PYTHON_REFRESH=quiet + +# Install Python and pip +RUN apt-get update && apt-get install -y --fix-broken && \ + apt-get install -y \ + python3 \ + python3-pip \ + python3-distutils \ + python3-setuptools \ + python3-wheel && \ + rm -rf /var/lib/apt/lists/* + +# Copy and install Python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Copy the rest of the application +COPY . . + +# Specify the entry point +CMD ["python3", "train.py", "with", "config_params_docker.json"] diff --git a/train/config_params_docker.json b/train/config_params_docker.json new file mode 100644 index 0000000..45f87d3 --- /dev/null +++ b/train/config_params_docker.json @@ -0,0 +1,54 @@ +{ + "backbone_type" : "nontransformer", + "task": "segmentation", + "n_classes" : 3, + "n_epochs" : 1, + "input_height" : 672, + "input_width" : 448, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "patches" : false, + "pretraining" : true, + "augmentation" : false, + "flip_aug" : false, + "blur_aug" : true, + "scaling" : true, + "adding_rgb_background": false, + "adding_rgb_foreground": false, + "add_red_textlines": false, + "channels_shuffling": true, + "degrading": true, + "brightening": true, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": true, + "transformer_num_patches_xy": [14, 21], + "transformer_patchsize_x": 1, + "transformer_patchsize_y": 1, + "transformer_projection_dim": 64, + "transformer_mlp_head_units": [128, 64], + "transformer_layers": 1, + "transformer_num_heads": 1, + "transformer_cnn_first": true, + "blur_k" : ["blur","gauss","median"], + "scales" : [0.6, 0.7, 0.8, 0.9], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "shuffle_indexes" : [ [0,2,1], [1,2,0], [1,0,2] , [2,1,0]], + "thetha" : [5, -5], + "number_of_backgrounds_per_image": 2, + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": true, + "data_is_provided": false, + "dir_train": "/entry_point_dir/train", + "dir_eval": "/entry_point_dir/eval", + "dir_output": "/entry_point_dir/output" +} diff --git a/train/train.py b/train/train.py index f6a4f47..e8e92af 100644 --- a/train/train.py +++ b/train/train.py @@ -53,7 +53,7 @@ def get_dirs_or_files(input_data): return image_input, labels_input -ex = Experiment() +ex = Experiment(save_git_info=False) @ex.config From 1b222594d694884108428d47a74aa67111d40218 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 25 Jun 2025 18:33:55 +0200 Subject: [PATCH 163/374] Update README.md: how to train model using docker image --- train/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/train/README.md b/train/README.md index b9e70a8..7c69a10 100644 --- a/train/README.md +++ b/train/README.md @@ -24,7 +24,19 @@ each class will be defined with a RGB value and beside images, a text file of cl ### Train To train a model, run: ``python train.py with config_params.json`` - + +### Train using Docker + +#### Build the Docker image + + ```bash + docker build -t model-training . + ``` +#### Run Docker image + ```bash + docker run --gpus all -v /host/path/to/entry_point_dir:/entry_point_dir model-training + ``` + ### Ground truth format Lables for each pixel are identified by a number. So if you have a binary case, ``n_classes`` should be set to ``2`` and labels should From 53dd4b26a95172f9aa33ff9806c637c18cad5ab4 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 3 Jul 2025 11:50:47 +0200 Subject: [PATCH 164/374] decorated with confidence value for cnnrnn ocr model --- src/eynollah/eynollah.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index fc60f2e..3b9d898 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5129,7 +5129,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_1225000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_step_900000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5487,7 +5487,7 @@ class Eynollah_ocr: if not self.export_textline_images_and_text: - if w_scaled < 640:#1.5*image_width: + if w_scaled < 750:#1.5*image_width: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) if abs(better_des_slope) > 45: @@ -5580,6 +5580,7 @@ class Eynollah_ocr: if not self.export_textline_images_and_text: extracted_texts = [] + extracted_conf_value = [] n_iterations = math.ceil(len(cropped_lines) / self.b_s) @@ -5700,12 +5701,19 @@ class Eynollah_ocr: preds_bin[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] preds = (preds + preds_bin) / 2. + pred_texts = decode_batch_predictions(preds, self.num_to_char) + + preds_max = np.max(preds, axis=2 ) + preds_max_args = np.argmax(preds, axis=2 ) + pred_max_not_unk_mask_bool = preds_max_args[:,:]!=256 + masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) for ib in range(imgs.shape[0]): pred_texts_ib = pred_texts[ib].replace("[UNK]", "") extracted_texts.append(pred_texts_ib) + extracted_conf_value.append(masked_means[ib]) del cropped_lines if self.prediction_with_both_of_rgb_and_bin: @@ -5713,7 +5721,10 @@ class Eynollah_ocr: gc.collect() extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + + extracted_conf_value_merged = [extracted_conf_value[ind] if cropped_lines_meging_indexing[ind]==0 else (extracted_conf_value[ind]+extracted_conf_value[ind+1])/2. if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + extracted_conf_value_merged = [extracted_conf_value_merged[ind_cfm] for ind_cfm in range(len(extracted_texts_merged)) if extracted_texts_merged[ind_cfm] is not None] extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) @@ -5791,6 +5802,7 @@ class Eynollah_ocr: if not is_textline_text: text_subelement = ET.SubElement(child_textregion, 'TextEquiv') + text_subelement.set('conf', f"{extracted_conf_value_merged[indexer]:.2f}") unicode_textline = ET.SubElement(text_subelement, 'Unicode') unicode_textline.text = extracted_texts_merged[indexer] else: @@ -5798,6 +5810,7 @@ class Eynollah_ocr: if childtest3.tag.endswith("TextEquiv"): for child_uc in childtest3: if child_uc.tag.endswith("Unicode"): + childtest3.set('conf', f"{extracted_conf_value_merged[indexer]:.2f}") child_uc.text = extracted_texts_merged[indexer] indexer = indexer + 1 From 04fead348fa612c36e428465e0df092dd701484c Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 3 Jul 2025 15:24:52 +0200 Subject: [PATCH 165/374] ocr: make sure that image height or width is not zero --- src/eynollah/eynollah.py | 4 ---- src/eynollah/utils/utils_ocr.py | 34 +++++++++++++++++++-------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 3b9d898..1260a96 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5435,7 +5435,6 @@ class Eynollah_ocr: mask_poly = mask_poly[y:y+h, x:x+w, :] img_crop = img_poly_on_img[y:y+h, x:x+w, :] - #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') if not self.do_not_mask_with_textline_contour: @@ -5482,9 +5481,6 @@ class Eynollah_ocr: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) - - - if not self.export_textline_images_and_text: if w_scaled < 750:#1.5*image_width: diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 81a8ae1..1e9162a 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -124,23 +124,26 @@ def return_textlines_split_if_needed(textline_image, textline_image_bin, predict else: return None, None def preprocess_and_resize_image_for_ocrcnn_model(img, image_height, image_width): - ratio = image_height /float(img.shape[0]) - w_ratio = int(ratio * img.shape[1]) - - if w_ratio <= image_width: - width_new = w_ratio + if img.shape[0]==0 or img.shape[1]==0: + img_fin = np.ones((image_height, image_width, 3)) else: - width_new = image_width + ratio = image_height /float(img.shape[0]) + w_ratio = int(ratio * img.shape[1]) - if width_new == 0: - width_new = img.shape[1] + if w_ratio <= image_width: + width_new = w_ratio + else: + width_new = image_width + + if width_new == 0: + width_new = img.shape[1] + - - img = resize_image(img, image_height, width_new) - img_fin = np.ones((image_height, image_width, 3))*255 + img = resize_image(img, image_height, width_new) + img_fin = np.ones((image_height, image_width, 3))*255 - img_fin[:,:width_new,:] = img[:,:,:] - img_fin = img_fin / 255. + img_fin[:,:width_new,:] = img[:,:,:] + img_fin = img_fin / 255. return img_fin def get_deskewed_contour_and_bb_and_image(contour, image, deskew_angle): @@ -188,7 +191,10 @@ def rotate_image_with_padding(image, angle, border_value=(0,0,0)): rotation_matrix[1, 2] += (new_h / 2) - center[1] # Perform the rotation - rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=border_value) + try: + rotated_image = cv2.warpAffine(image, rotation_matrix, (new_w, new_h), borderValue=border_value) + except: + rotated_image = np.copy(image) return rotated_image From fee40049cdfe1325d65f717b66fe3ccc11d4c9d4 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 16 Jul 2025 14:00:12 +0200 Subject: [PATCH 166/374] ocr model renamed - image text font for ocr result is now using Charis-7.000 font (downloaded from here https://software.sil.org/charis/download/) --- src/eynollah/eynollah.py | 148 +++++++++++++++++++++------------------ 1 file changed, 78 insertions(+), 70 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 1260a96..bf11dec 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -318,7 +318,7 @@ class Eynollah: if self.ocr and self.tr: self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" elif self.ocr and not self.tr: - self.model_ocr_dir = dir_models + "/model_step_750000_ocr"#"/model_step_125000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250716" if self.tables: if self.light_version: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" @@ -5129,7 +5129,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_900000_ocr"#"/model_step_25000_ocr"#"/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250716" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5276,7 +5276,7 @@ class Eynollah_ocr: if self.draw_texts_on_image: - font_path = "NotoSans-Regular.ttf" # Make sure this file exists! + font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = ImageFont.truetype(font_path, 40) for indexer_text, bb_ind in enumerate(total_bb_coordinates): @@ -5340,8 +5340,8 @@ class Eynollah_ocr: tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) #print("Job done in %.1fs", time.time() - t0) else: - max_len = 512 - padding_token = 299 + max_len = 512#280#512 + padding_token = 299#1500#299 image_width = 512#max_len * 4 image_height = 32 @@ -5435,52 +5435,57 @@ class Eynollah_ocr: mask_poly = mask_poly[y:y+h, x:x+w, :] img_crop = img_poly_on_img[y:y+h, x:x+w, :] - - #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') - if not self.do_not_mask_with_textline_contour: - if angle_degrees > 3: - better_des_slope = get_orientation_moments(textline_coords) - - img_crop = rotate_image_with_padding(img_crop, better_des_slope ) - - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) - - mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) - mask_poly = mask_poly.astype('uint8') - - #new bounding box - x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_poly[:,:,0]) - - mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] - img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] - + + if self.export_textline_images_and_text: + if not self.do_not_mask_with_textline_contour: img_crop[mask_poly==0] = 255 - - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] - img_crop_bin[mask_poly==0] = 255 - - if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: + + else: + #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') + if not self.do_not_mask_with_textline_contour: + if angle_degrees > 3: + better_des_slope = get_orientation_moments(textline_coords) + + img_crop = rotate_image_with_padding(img_crop, better_des_slope ) + if self.prediction_with_both_of_rgb_and_bin: - img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) - else: - img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) - + img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) + + mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) + mask_poly = mask_poly.astype('uint8') - else: - better_des_slope = 0 - img_crop[mask_poly==0] = 255 - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin[mask_poly==0] = 255 - if type_textregion=='drop-capital': - pass - else: - if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: + #new bounding box + x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_poly[:,:,0]) + + mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] + + img_crop[mask_poly==0] = 255 + + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_crop_bin[mask_poly==0] = 255 + + if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: if self.prediction_with_both_of_rgb_and_bin: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + + + else: + better_des_slope = 0 + img_crop[mask_poly==0] = 255 + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin[mask_poly==0] = 255 + if type_textregion=='drop-capital': + pass + else: + if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: + if self.prediction_with_both_of_rgb_and_bin: + img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) + else: + img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) if not self.export_textline_images_and_text: if w_scaled < 750:#1.5*image_width: @@ -5541,35 +5546,38 @@ class Eynollah_ocr: cropped_lines_bin.append(img_fin) if self.export_textline_images_and_text: - if child_textlines.tag.endswith("TextEquiv"): - for cheild_text in child_textlines: - if cheild_text.tag.endswith("Unicode"): - textline_text = cheild_text.text - if textline_text: - if self.do_not_mask_with_textline_contour: - if self.pref_of_dataset: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.txt'), 'w') as text_file: - text_file.write(textline_text) + if img_crop.shape[0]==0 or img_crop.shape[1]==0: + pass + else: + if child_textlines.tag.endswith("TextEquiv"): + for cheild_text in child_textlines: + if cheild_text.tag.endswith("Unicode"): + textline_text = cheild_text.text + if textline_text: + if self.do_not_mask_with_textline_contour: + if self.pref_of_dataset: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.txt'), 'w') as text_file: + text_file.write(textline_text) - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.png'), img_crop ) + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.png'), img_crop ) + else: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.txt'), 'w') as text_file: + text_file.write(textline_text) + + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.png'), img_crop ) else: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.txt'), 'w') as text_file: - text_file.write(textline_text) + if self.pref_of_dataset: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.txt'), 'w') as text_file: + text_file.write(textline_text) - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.png'), img_crop ) - else: - if self.pref_of_dataset: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.txt'), 'w') as text_file: - text_file.write(textline_text) + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.png'), img_crop ) + else: + with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.txt'), 'w') as text_file: + text_file.write(textline_text) - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.png'), img_crop ) - else: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.txt'), 'w') as text_file: - text_file.write(textline_text) - - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.png'), img_crop ) - - indexer_textlines+=1 + cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.png'), img_crop ) + + indexer_textlines+=1 if not self.export_textline_images_and_text: indexer_text_region = indexer_text_region +1 @@ -5727,7 +5735,7 @@ class Eynollah_ocr: if self.draw_texts_on_image: - font_path = "NotoSans-Regular.ttf" # Make sure this file exists! + font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = ImageFont.truetype(font_path, 40) for indexer_text, bb_ind in enumerate(total_bb_coordinates): From 673e67a847935c3ff3dd15cf2c67095aae36ecb8 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 21 Jul 2025 10:54:20 +0200 Subject: [PATCH 167/374] update model names --- src/eynollah/eynollah.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index bf11dec..12acff7 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5129,7 +5129,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250716" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250716"#"/model_ens_ocrcnn_new6"#"/model_ens_ocrcnn_new2"# model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5143,7 +5143,6 @@ class Eynollah_ocr: with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: characters = json.load(config_file) - AUTOTUNE = tf.data.AUTOTUNE @@ -5154,6 +5153,7 @@ class Eynollah_ocr: self.num_to_char = StringLookup( vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True ) + self.end_character = len(characters) + 2 def run(self, overwrite : bool = False): if self.dir_in: @@ -5340,8 +5340,8 @@ class Eynollah_ocr: tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) #print("Job done in %.1fs", time.time() - t0) else: - max_len = 512#280#512 - padding_token = 299#1500#299 + ###max_len = 280#512#280#512 + ###padding_token = 1500#299#1500#299 image_width = 512#max_len * 4 image_height = 32 @@ -5656,13 +5656,13 @@ class Eynollah_ocr: preds_flipped = self.prediction_model.predict(imgs_ver_flipped, verbose=0) preds_max_fliped = np.max(preds_flipped, axis=2 ) preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) - pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=256 + pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=self.end_character masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) masked_means_flipped[np.isnan(masked_means_flipped)] = 0 preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) - pred_max_not_unk_mask_bool = preds_max_args[:,:]!=256 + pred_max_not_unk_mask_bool = preds_max_args[:,:]!=self.end_character masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) masked_means[np.isnan(masked_means)] = 0 @@ -5683,13 +5683,13 @@ class Eynollah_ocr: preds_flipped = self.prediction_model.predict(imgs_bin_ver_flipped, verbose=0) preds_max_fliped = np.max(preds_flipped, axis=2 ) preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) - pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=256 + pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=self.end_character masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) masked_means_flipped[np.isnan(masked_means_flipped)] = 0 preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) - pred_max_not_unk_mask_bool = preds_max_args[:,:]!=256 + pred_max_not_unk_mask_bool = preds_max_args[:,:]!=self.end_character masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) masked_means[np.isnan(masked_means)] = 0 @@ -5711,7 +5711,7 @@ class Eynollah_ocr: preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) - pred_max_not_unk_mask_bool = preds_max_args[:,:]!=256 + pred_max_not_unk_mask_bool = preds_max_args[:,:]!=self.end_character masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) for ib in range(imgs.shape[0]): From daa597dbaaa12be3d2435960fb272852fc89c09a Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 21 Jul 2025 14:50:05 +0200 Subject: [PATCH 168/374] should merged text for the whole page be written in xml? --- src/eynollah/eynollah.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 12acff7..bdb8f1a 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5129,7 +5129,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250716"#"/model_ens_ocrcnn_new6"#"/model_ens_ocrcnn_new2"# + self.model_ocr_dir = dir_models + "/model_ens_ocrcnn_new6"#"/model_eynollah_ocr_cnnrnn_20250716"#"/model_ens_ocrcnn_new6"#"/model_ens_ocrcnn_new2"# model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5141,7 +5141,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) - with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: + with open(os.path.join(self.model_ocr_dir, "characters_20250707_all_lang.txt"),"r") as config_file: characters = json.load(config_file) AUTOTUNE = tf.data.AUTOTUNE @@ -5780,9 +5780,24 @@ class Eynollah_ocr: text_by_textregion.append(" ".join(extracted_texts_merged_un)) #print(text_by_textregion, 'text_by_textregiontext_by_textregiontext_by_textregiontext_by_textregiontext_by_textregion') + + ###index_tot_regions = [] + ###tot_region_ref = [] + + ###for jj in root1.iter(link+'RegionRefIndexed'): + ###index_tot_regions.append(jj.attrib['index']) + ###tot_region_ref.append(jj.attrib['regionRef']) + + ###id_to_order = {tid: ro for tid, ro in zip(tot_region_ref, index_tot_regions)} + + id_textregions = [] + textregions_by_existing_ids = [] indexer = 0 indexer_textregion = 0 for nn in root1.iter(region_tags): + id_textregion = nn.attrib['id'] + id_textregions.append(id_textregion) + textregions_by_existing_ids.append(text_by_textregion[indexer_textregion]) is_textregion_text = False for childtest in nn: @@ -5829,7 +5844,17 @@ class Eynollah_ocr: else: unicode_textregion.text = text_by_textregion[indexer_textregion] indexer_textregion = indexer_textregion + 1 - + + ###sample_order = [(id_to_order[tid], text) for tid, text in zip(id_textregions, textregions_by_existing_ids) if tid in id_to_order] + + ##ordered_texts_sample = [text for _, text in sorted(sample_order)] + ##tot_page_text = ' '.join(ordered_texts_sample) + + ##for page_element in root1.iter(link+'Page'): + ##text_page = ET.SubElement(page_element, 'TextEquiv') + ##unicode_textpage = ET.SubElement(text_page, 'Unicode') + ##unicode_textpage.text = tot_page_text + ET.register_namespace("",name_space) tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) #print("Job done in %.1fs", time.time() - t0) From da141bb42e6f7af4a069a77942e0695c68a56592 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 23 Jul 2025 16:44:17 +0200 Subject: [PATCH 169/374] resolving tests error --- tests/test_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 607140e..b4e2dbd 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -85,8 +85,8 @@ def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, ca outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') args = [ '-m', SBBBIN_MODELS, - str(infile), - str(outfile), + '-i', str(infile), + '-o', str(outfile), ] caplog.set_level(logging.INFO) def only_eynollah(logrec): @@ -117,7 +117,7 @@ def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, c args = [ '-m', SBBBIN_MODELS, '-di', str(indir), - '-do', str(outdir), + '-o', str(outdir), ] caplog.set_level(logging.INFO) def only_eynollah(logrec): From fd0595f9207fb2f608eb1ae3c40dc6826a409d38 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 24 Jul 2025 13:52:38 +0200 Subject: [PATCH 170/374] Update Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f2bf34..1458427 100644 --- a/Makefile +++ b/Makefile @@ -85,7 +85,7 @@ smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif eynollah layout -di $( Date: Fri, 25 Jul 2025 13:18:38 +0200 Subject: [PATCH 171/374] threshold for textline ocr + new ocr model --- src/eynollah/cli.py | 8 ++- src/eynollah/eynollah.py | 117 +++++++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 49 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 9398c47..a313860 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -496,6 +496,11 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ "-ds_pref", help="in the case of extracting textline and text from a xml GT file user can add an abbrevation of dataset name to generated dataset", ) +@click.option( + "--min_conf_value_of_textline_text", + "-min_conf", + help="minimum OCR confidence value. Text lines with a confidence value lower than this threshold will not be included in the output XML file.", +) @click.option( "--log_level", "-l", @@ -503,7 +508,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, log_level): +def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) @@ -530,6 +535,7 @@ def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, prediction_with_both_of_rgb_and_bin=prediction_with_both_of_rgb_and_bin, batch_size=batch_size, pref_of_dataset=dataset_abbrevation, + min_conf_value_of_textline_text=min_conf_value_of_textline_text, ) eynollah_ocr.run(overwrite=overwrite) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index bdb8f1a..aa1b2e1 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -318,7 +318,7 @@ class Eynollah: if self.ocr and self.tr: self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" elif self.ocr and not self.tr: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250716" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250725" if self.tables: if self.light_version: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" @@ -4974,13 +4974,23 @@ class Eynollah: gc.collect() if len(all_found_textline_polygons)>0: ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + else: + ocr_all_textlines = None + if all_found_textline_polygons_marginals and len(all_found_textline_polygons_marginals)>0: ocr_all_textlines_marginals = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + else: + ocr_all_textlines_marginals = None if all_found_textline_polygons_h and len(all_found_textline_polygons)>0: ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_h, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + else: + ocr_all_textlines_h = None + if polygons_of_drop_capitals and len(polygons_of_drop_capitals)>0: ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines(image_page, polygons_of_drop_capitals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + else: + ocr_all_textlines_drop = None else: ocr_all_textlines = None ocr_all_textlines_marginals = None @@ -5098,7 +5108,8 @@ class Eynollah_ocr: do_not_mask_with_textline_contour=False, draw_texts_on_image=False, prediction_with_both_of_rgb_and_bin=False, - pref_of_dataset = None, + pref_of_dataset=None, + min_conf_value_of_textline_text : Optional[float]=None, logger=None, ): self.dir_in = dir_in @@ -5117,6 +5128,10 @@ class Eynollah_ocr: self.logger = logger if logger else getLogger('eynollah') if not export_textline_images_and_text: + if min_conf_value_of_textline_text: + self.min_conf_value_of_textline_text = float(min_conf_value_of_textline_text) + else: + self.min_conf_value_of_textline_text = 0.3 if tr_ocr: self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") @@ -5129,7 +5144,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_ens_ocrcnn_new6"#"/model_eynollah_ocr_cnnrnn_20250716"#"/model_ens_ocrcnn_new6"#"/model_ens_ocrcnn_new2"# + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250725"#"/model_step_1020000_ocr"#"/model_ens_ocrcnn_new10"#"/model_step_255000_ocr"#"/model_ens_ocrcnn_new9"#"/model_step_900000_ocr"#"/model_eynollah_ocr_cnnrnn_20250716"#"/model_ens_ocrcnn_new6"#"/model_ens_ocrcnn_new2"# model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5139,9 +5154,8 @@ class Eynollah_ocr: self.b_s = 8 else: self.b_s = int(batch_size) - - with open(os.path.join(self.model_ocr_dir, "characters_20250707_all_lang.txt"),"r") as config_file: + with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: characters = json.load(config_file) AUTOTUNE = tf.data.AUTOTUNE @@ -5442,50 +5456,54 @@ class Eynollah_ocr: else: #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') - if not self.do_not_mask_with_textline_contour: - if angle_degrees > 3: - better_des_slope = get_orientation_moments(textline_coords) + + if angle_degrees > 3: + better_des_slope = get_orientation_moments(textline_coords) + + img_crop = rotate_image_with_padding(img_crop, better_des_slope ) + + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) - img_crop = rotate_image_with_padding(img_crop, better_des_slope ) + mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) + mask_poly = mask_poly.astype('uint8') + + #new bounding box + x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_poly[:,:,0]) + + mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] + img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) - - mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) - mask_poly = mask_poly.astype('uint8') - - #new bounding box - x_n, y_n, w_n, h_n = get_contours_and_bounding_boxes(mask_poly[:,:,0]) - - mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :] - img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :] - + if not self.do_not_mask_with_textline_contour: img_crop[mask_poly==0] = 255 - - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] + + if self.prediction_with_both_of_rgb_and_bin: + img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] + if not self.do_not_mask_with_textline_contour: img_crop_bin[mask_poly==0] = 255 + + if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: + if self.prediction_with_both_of_rgb_and_bin: + img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) + else: + img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + - if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: + else: + better_des_slope = 0 + if not self.do_not_mask_with_textline_contour: + img_crop[mask_poly==0] = 255 + if self.prediction_with_both_of_rgb_and_bin: + if not self.do_not_mask_with_textline_contour: + img_crop_bin[mask_poly==0] = 255 + if type_textregion=='drop-capital': + pass + else: + if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: if self.prediction_with_both_of_rgb_and_bin: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) - - - else: - better_des_slope = 0 - img_crop[mask_poly==0] = 255 - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin[mask_poly==0] = 255 - if type_textregion=='drop-capital': - pass - else: - if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: - if self.prediction_with_both_of_rgb_and_bin: - img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) - else: - img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) if not self.export_textline_images_and_text: if w_scaled < 750:#1.5*image_width: @@ -5716,9 +5734,12 @@ class Eynollah_ocr: for ib in range(imgs.shape[0]): pred_texts_ib = pred_texts[ib].replace("[UNK]", "") - extracted_texts.append(pred_texts_ib) - extracted_conf_value.append(masked_means[ib]) - + if masked_means[ib] >= self.min_conf_value_of_textline_text: + extracted_texts.append(pred_texts_ib) + extracted_conf_value.append(masked_means[ib]) + else: + extracted_texts.append("") + extracted_conf_value.append(0) del cropped_lines if self.prediction_with_both_of_rgb_and_bin: del cropped_lines_bin @@ -5790,14 +5811,14 @@ class Eynollah_ocr: ###id_to_order = {tid: ro for tid, ro in zip(tot_region_ref, index_tot_regions)} - id_textregions = [] - textregions_by_existing_ids = [] + #id_textregions = [] + #textregions_by_existing_ids = [] indexer = 0 indexer_textregion = 0 for nn in root1.iter(region_tags): - id_textregion = nn.attrib['id'] - id_textregions.append(id_textregion) - textregions_by_existing_ids.append(text_by_textregion[indexer_textregion]) + #id_textregion = nn.attrib['id'] + #id_textregions.append(id_textregion) + #textregions_by_existing_ids.append(text_by_textregion[indexer_textregion]) is_textregion_text = False for childtest in nn: From 322b04145f7b1460dfe9a3fbd702e3c65dd29ca3 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 5 Aug 2025 14:22:22 +0200 Subject: [PATCH 172/374] use the latest ocr model with balanced fraktur-antiqua training dataset --- src/eynollah/cli.py | 4 ++-- src/eynollah/eynollah.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index a313860..5135534 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -325,12 +325,12 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low @click.option( "--threshold_art_class_layout", "-tharl", - help="threshold of artifical class in the case of layout detection", + help="threshold of artifical class in the case of layout detection. The default value is 0.1", ) @click.option( "--threshold_art_class_textline", "-thart", - help="threshold of artifical class in the case of textline detection", + help="threshold of artifical class in the case of textline detection. The default value is 0.1", ) @click.option( "--skip_layout_and_reading_order", diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index aa1b2e1..9e5ba51 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -318,7 +318,7 @@ class Eynollah: if self.ocr and self.tr: self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" elif self.ocr and not self.tr: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250725" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250805" if self.tables: if self.light_version: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" @@ -5144,7 +5144,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250725"#"/model_step_1020000_ocr"#"/model_ens_ocrcnn_new10"#"/model_step_255000_ocr"#"/model_ens_ocrcnn_new9"#"/model_step_900000_ocr"#"/model_eynollah_ocr_cnnrnn_20250716"#"/model_ens_ocrcnn_new6"#"/model_ens_ocrcnn_new2"# + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250805" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( From 6462ea5b33cd6e4c1eaac1b2bf1fe072147e76f9 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 6 Aug 2025 22:33:42 +0200 Subject: [PATCH 173/374] adding visualization of ocr text of xml file --- train/generate_gt_for_training.py | 81 +++++++++++++++++++++++++++++++ train/gt_gen_utils.py | 71 +++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 8ca5cd3..1971f68 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -3,6 +3,7 @@ import json from gt_gen_utils import * from tqdm import tqdm from pathlib import Path +from PIL import Image, ImageDraw, ImageFont @click.group() def main(): @@ -447,6 +448,86 @@ def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): cv2.imwrite(os.path.join(dir_out, f_name+'.png'), added_image) + + +@main.command() +@click.option( + "--xml_file", + "-xml", + help="xml filename", + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--dir_xml", + "-dx", + help="directory of GT page-xml files", + type=click.Path(exists=True, file_okay=False), +) + +@click.option( + "--dir_out", + "-do", + help="directory where plots will be written", + type=click.Path(exists=True, file_okay=False), +) + + +def visualize_ocr_text(xml_file, dir_xml, dir_out): + assert xml_file or dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" + if dir_xml: + xml_files_ind = os.listdir(dir_xml) + else: + xml_files_ind = [xml_file] + + font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! + font = ImageFont.truetype(font_path, 40) + + for ind_xml in tqdm(xml_files_ind): + indexer = 0 + #print(ind_xml) + #print('########################') + if dir_xml: + xml_file = os.path.join(dir_xml,ind_xml ) + f_name = Path(ind_xml).stem + else: + xml_file = os.path.join(ind_xml ) + f_name = Path(ind_xml).stem + print(f_name, 'f_name') + + co_tetxlines, y_len, x_len, ocr_texts = get_textline_contours_and_ocr_text(xml_file) + + total_bb_coordinates = [] + + image_text = Image.new("RGB", (x_len, y_len), "white") + draw = ImageDraw.Draw(image_text) + + + + for index, cnt in enumerate(co_tetxlines): + x,y,w,h = cv2.boundingRect(cnt) + #total_bb_coordinates.append([x,y,w,h]) + + #fit_text_single_line + + #x_bb = bb_ind[0] + #y_bb = bb_ind[1] + #w_bb = bb_ind[2] + #h_bb = bb_ind[3] + + font = fit_text_single_line(draw, ocr_texts[index], font_path, w, int(h*0.4) ) + + ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) + + text_bbox = draw.textbbox((0, 0), ocr_texts[index], font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + + text_x = x + (w - text_width) // 2 # Center horizontally + text_y = y + (h - text_height) // 2 # Center vertically + + # Draw the text + draw.text((text_x, text_y), ocr_texts[index], fill="black", font=font) + image_text.save(os.path.join(dir_out, f_name+'.png')) if __name__ == "__main__": main() diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 5076dd6..907e04d 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -9,6 +9,7 @@ import cv2 from shapely import geometry from pathlib import Path import matplotlib.pyplot as plt +from PIL import Image, ImageDraw, ImageFont KERNEL = np.ones((5, 5), np.uint8) @@ -283,6 +284,76 @@ def get_textline_contours_for_visualization(xml_file): return co_use_case, y_len, x_len +def get_textline_contours_and_ocr_text(xml_file): + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + + region_tags = np.unique([x for x in alltags if x.endswith('TextLine')]) + tag_endings = ['}TextLine','}textline'] + co_use_case = [] + ocr_textlines = [] + + for tag in region_tags: + if tag.endswith(tag_endings[0]) or tag.endswith(tag_endings[1]): + for nn in root1.iter(tag): + c_t_in = [] + ocr_text_in = [''] + sumi = 0 + for vv in nn.iter(): + if vv.tag == link + 'Coords': + for childtest2 in nn: + if childtest2.tag.endswith("TextEquiv"): + for child_uc in childtest2: + if child_uc.tag.endswith("Unicode"): + text = child_uc.text + ocr_text_in[0]= text + + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + + + if vv.tag == link + 'Point': + c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + + + co_use_case.append(np.array(c_t_in)) + ocr_textlines.append(ocr_text_in[0]) + return co_use_case, y_len, x_len, ocr_textlines + +def fit_text_single_line(draw, text, font_path, max_width, max_height): + initial_font_size = 50 + font_size = initial_font_size + while font_size > 10: # Minimum font size + font = ImageFont.truetype(font_path, font_size) + text_bbox = draw.textbbox((0, 0), text, font=font) # Get text bounding box + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + + if text_width <= max_width and text_height <= max_height: + return font # Return the best-fitting font + + font_size -= 2 # Reduce font size and retry + + return ImageFont.truetype(font_path, 10) # Smallest font fallback + def get_layout_contours_for_visualization(xml_file): tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) root1=tree1.getroot() From 263da755ef5d1a03f6398d090b02a094025a52aa Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 7 Aug 2025 10:32:49 +0200 Subject: [PATCH 174/374] loading xmls with UTF-8 encoding --- train/generate_gt_for_training.py | 26 +++++++++++++------------- train/gt_gen_utils.py | 10 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 1971f68..d4b58dc 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -495,7 +495,7 @@ def visualize_ocr_text(xml_file, dir_xml, dir_out): print(f_name, 'f_name') co_tetxlines, y_len, x_len, ocr_texts = get_textline_contours_and_ocr_text(xml_file) - + total_bb_coordinates = [] image_text = Image.new("RGB", (x_len, y_len), "white") @@ -513,20 +513,20 @@ def visualize_ocr_text(xml_file, dir_xml, dir_out): #y_bb = bb_ind[1] #w_bb = bb_ind[2] #h_bb = bb_ind[3] - - font = fit_text_single_line(draw, ocr_texts[index], font_path, w, int(h*0.4) ) - - ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) - - text_bbox = draw.textbbox((0, 0), ocr_texts[index], font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] + if ocr_texts[index]: + font = fit_text_single_line(draw, ocr_texts[index], font_path, w, int(h*0.4) ) + + ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) + + text_bbox = draw.textbbox((0, 0), ocr_texts[index], font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] - text_x = x + (w - text_width) // 2 # Center horizontally - text_y = y + (h - text_height) // 2 # Center vertically + text_x = x + (w - text_width) // 2 # Center horizontally + text_y = y + (h - text_height) // 2 # Center vertically - # Draw the text - draw.text((text_x, text_y), ocr_texts[index], fill="black", font=font) + # Draw the text + draw.text((text_x, text_y), ocr_texts[index], fill="black", font=font) image_text.save(os.path.join(dir_out, f_name+'.png')) if __name__ == "__main__": diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 907e04d..753b0f5 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -244,7 +244,7 @@ def update_region_contours(co_text, img_boundary, erosion_rate, dilation_rate, y return co_text_eroded, img_boundary def get_textline_contours_for_visualization(xml_file): - tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' @@ -285,7 +285,7 @@ def get_textline_contours_for_visualization(xml_file): def get_textline_contours_and_ocr_text(xml_file): - tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' @@ -355,7 +355,7 @@ def fit_text_single_line(draw, text, font_path, max_width, max_height): return ImageFont.truetype(font_path, 10) # Smallest font fallback def get_layout_contours_for_visualization(xml_file): - tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' @@ -630,7 +630,7 @@ def get_images_of_ground_truth(gt_list, dir_in, output_dir, output_type, config_ for index in tqdm(range(len(gt_list))): #try: print(gt_list[index]) - tree1 = ET.parse(dir_in+'/'+gt_list[index], parser = ET.XMLParser(encoding = 'iso-8859-5')) + tree1 = ET.parse(dir_in+'/'+gt_list[index], parser = ET.XMLParser(encoding='utf-8')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' @@ -1311,7 +1311,7 @@ def find_new_features_of_contours(contours_main): return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin def read_xml(xml_file): file_name = Path(xml_file).stem - tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding = 'iso-8859-5')) + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] link=alltags[0].split('}')[0]+'}' From 52d9cc9bafe5021d93999e975703fa0ad315337a Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 8 Aug 2025 11:32:02 +0200 Subject: [PATCH 175/374] deskewing with faster multiprocessing --- src/eynollah/eynollah.py | 9 +-- src/eynollah/utils/separate_lines.py | 103 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9e5ba51..5299d3e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -96,6 +96,7 @@ from .utils.separate_lines import ( textline_contours_postprocessing, separate_lines_new2, return_deskew_slop, + return_deskew_slop_old_mp, do_work_of_slopes_new, do_work_of_slopes_new_curved, do_work_of_slopes_new_light, @@ -1936,8 +1937,8 @@ class Eynollah: y_diff_mean = find_contours_mean_y_diff(textline_con_fil) sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) crop_img[crop_img > 0] = 1 - slope_corresponding_textregion = return_deskew_slop(crop_img, sigma_des, - map=self.executor.map, logger=self.logger, plotter=self.plotter) + slope_corresponding_textregion = return_deskew_slop_old_mp(crop_img, sigma_des, + logger=self.logger, plotter=self.plotter) except Exception as why: self.logger.error(why) slope_corresponding_textregion = MAX_SLOPE @@ -3203,8 +3204,8 @@ class Eynollah: def run_deskew(self, textline_mask_tot_ea): #print(textline_mask_tot_ea.shape, 'textline_mask_tot_ea deskew') - slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True, - map=self.executor.map, logger=self.logger, plotter=self.plotter) + slope_deskew = return_deskew_slop_old_mp(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True, + logger=self.logger, plotter=self.plotter) slope_first = 0 if self.plotter: diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 6289d4d..ead5cfb 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -5,6 +5,8 @@ import numpy as np import cv2 from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d +from multiprocessing import Process, Queue, cpu_count +from multiprocessing import Pool from .rotate import rotate_image from .resize import resize_image from .contour import ( @@ -1526,6 +1528,107 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map angle = 0 return angle + +def return_deskew_slop_old_mp(img_patch_org, sigma_des,n_tot_angles=100, + main_page=False, logger=None, plotter=None): + if main_page and plotter: + plotter.save_plot_of_textline_density(img_patch_org) + + img_int=np.zeros((img_patch_org.shape[0],img_patch_org.shape[1])) + img_int[:,:]=img_patch_org[:,:]#img_patch_org[:,:,0] + + max_shape=np.max(img_int.shape) + img_resized=np.zeros((int( max_shape*(1.1) ) , int( max_shape*(1.1) ) )) + + onset_x=int((img_resized.shape[1]-img_int.shape[1])/2.) + onset_y=int((img_resized.shape[0]-img_int.shape[0])/2.) + + img_resized[ onset_y:onset_y+img_int.shape[0] , onset_x:onset_x+img_int.shape[1] ]=img_int[:,:] + + if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]: + angles = np.array([-45, 0, 45, 90,]) + angle = get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=plotter) + + angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles) + angle = get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=plotter) + elif main_page: + angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) + angle = get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=plotter) + + early_slope_edge=11 + if abs(angle) > early_slope_edge: + if angle < 0: + angles = np.linspace(-90, -12, n_tot_angles) + else: + angles = np.linspace(90, 12, n_tot_angles) + angle = get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=plotter) + else: + angles = np.linspace(-25, 25, int(0.5 * n_tot_angles) + 10) + angle = get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=plotter) + + early_slope_edge=22 + if abs(angle) > early_slope_edge: + if angle < 0: + angles = np.linspace(-90, -25, int(0.5 * n_tot_angles) + 10) + else: + angles = np.linspace(90, 25, int(0.5 * n_tot_angles) + 10) + angle = get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=plotter) + + return angle + +def do_image_rotation_omp(queue_of_all_params,angles_per_process, img_resized, sigma_des): + vars_per_each_subprocess = [] + angles_per_each_subprocess = [] + for mv in range(len(angles_per_process)): + img_rot=rotate_image(img_resized,angles_per_process[mv]) + img_rot[img_rot!=0]=1 + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + vars_per_each_subprocess.append(var_spectrum) + angles_per_each_subprocess.append(angles_per_process[mv]) + + queue_of_all_params.put([vars_per_each_subprocess, angles_per_each_subprocess]) + +def get_smallest_skew_omp(img_resized, sigma_des, angles, plotter=None): + num_cores = cpu_count() + + queue_of_all_params = Queue() + processes = [] + nh = np.linspace(0, len(angles), num_cores + 1) + + for i in range(num_cores): + angles_per_process = angles[int(nh[i]) : int(nh[i + 1])] + processes.append(Process(target=do_image_rotation_omp, args=(queue_of_all_params, angles_per_process, img_resized, sigma_des))) + + for i in range(num_cores): + processes[i].start() + + var_res=[] + all_angles = [] + for i in range(num_cores): + list_all_par = queue_of_all_params.get(True) + vars_for_subprocess = list_all_par[0] + angles_sub_process = list_all_par[1] + for j in range(len(vars_for_subprocess)): + var_res.append(vars_for_subprocess[j]) + all_angles.append(angles_sub_process[j]) + + for i in range(num_cores): + processes[i].join() + + if plotter: + plotter.save_plot_of_rotation_angle(all_angles, var_res) + + + try: + var_res=np.array(var_res) + ang_int=all_angles[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + return ang_int + def do_work_of_slopes_new( box_text, contour, contour_par, index_r_con, textline_mask_tot_ea, image_page_rotated, slope_deskew, From cf4983da54a1d8e0e5e382569a5502110b438189 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 8 Aug 2025 16:12:55 +0200 Subject: [PATCH 176/374] visualize vertical ocr text vertically --- train/generate_gt_for_training.py | 36 +++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index d4b58dc..91ee2c8 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -514,19 +514,37 @@ def visualize_ocr_text(xml_file, dir_xml, dir_out): #w_bb = bb_ind[2] #h_bb = bb_ind[3] if ocr_texts[index]: + + + is_vertical = h > 2*w # Check orientation font = fit_text_single_line(draw, ocr_texts[index], font_path, w, int(h*0.4) ) - ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) - - text_bbox = draw.textbbox((0, 0), ocr_texts[index], font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] + if is_vertical: + + vertical_font = fit_text_single_line(draw, ocr_texts[index], font_path, h, int(w * 0.8)) - text_x = x + (w - text_width) // 2 # Center horizontally - text_y = y + (h - text_height) // 2 # Center vertically + text_img = Image.new("RGBA", (h, w), (255, 255, 255, 0)) # Note: dimensions are swapped + text_draw = ImageDraw.Draw(text_img) + text_draw.text((0, 0), ocr_texts[index], font=vertical_font, fill="black") - # Draw the text - draw.text((text_x, text_y), ocr_texts[index], fill="black", font=font) + # Rotate text image by 90 degrees + rotated_text = text_img.rotate(90, expand=1) + + # Calculate paste position (centered in bbox) + paste_x = x + (w - rotated_text.width) // 2 + paste_y = y + (h - rotated_text.height) // 2 + + image_text.paste(rotated_text, (paste_x, paste_y), rotated_text) # Use rotated image as mask + else: + text_bbox = draw.textbbox((0, 0), ocr_texts[index], font=font) + text_width = text_bbox[2] - text_bbox[0] + text_height = text_bbox[3] - text_bbox[1] + + text_x = x + (w - text_width) // 2 # Center horizontally + text_y = y + (h - text_height) // 2 # Center vertically + + # Draw the text + draw.text((text_x, text_y), ocr_texts[index], fill="black", font=font) image_text.save(os.path.join(dir_out, f_name+'.png')) if __name__ == "__main__": From 268aa141d7b70a63e5b2ef317fda864249f8f17c Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 12 Aug 2025 12:50:15 +0200 Subject: [PATCH 177/374] avoiding float in range --- src/eynollah/utils/__init__.py | 41 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 7fa4a7b..ca86047 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1801,8 +1801,8 @@ def return_boxes_of_images_by_order_of_reading_new( #print(y_type_2_up,x_starting_up,x_ending_up,'didid') nodes_in = [] for ij in range(len(x_starting_up)): - nodes_in = nodes_in + list(range(x_starting_up[ij], - x_ending_up[ij])) + nodes_in = nodes_in + list(range(int(x_starting_up[ij]), + int(x_ending_up[ij]))) nodes_in = np.unique(nodes_in) #print(nodes_in,'nodes_in') @@ -1825,8 +1825,8 @@ def return_boxes_of_images_by_order_of_reading_new( elif len(y_diff_main_separator_up)==0: nodes_in = [] for ij in range(len(x_starting_up)): - nodes_in = nodes_in + list(range(x_starting_up[ij], - x_ending_up[ij])) + nodes_in = nodes_in + list(range(int(x_starting_up[ij]), + int(x_ending_up[ij]))) nodes_in = np.unique(nodes_in) #print(nodes_in,'nodes_in2') #print(np.array(range(len(peaks_neg_tot)-1)),'np.array(range(len(peaks_neg_tot)-1))') @@ -1866,8 +1866,8 @@ def return_boxes_of_images_by_order_of_reading_new( columns_covered_by_mothers = [] for dj in range(len(x_start_without_mother)): columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_start_without_mother[dj], - x_end_without_mother[dj])) + list(range(int(x_start_without_mother[dj]), + int(x_end_without_mother[dj]))) columns_covered_by_mothers = list(set(columns_covered_by_mothers)) all_columns=np.arange(len(peaks_neg_tot)-1) @@ -1909,8 +1909,8 @@ def return_boxes_of_images_by_order_of_reading_new( columns_covered_by_mothers = [] for dj in range(len(x_start_without_mother)): columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_start_without_mother[dj], - x_end_without_mother[dj])) + list(range(int(x_start_without_mother[dj]), + int(x_end_without_mother[dj]))) columns_covered_by_mothers = list(set(columns_covered_by_mothers)) all_columns=np.arange(len(peaks_neg_tot)-1) @@ -1926,8 +1926,8 @@ def return_boxes_of_images_by_order_of_reading_new( columns_covered_by_with_child_no_mothers = [] for dj in range(len(x_end_with_child_without_mother)): columns_covered_by_with_child_no_mothers = columns_covered_by_with_child_no_mothers + \ - list(range(x_start_with_child_without_mother[dj], - x_end_with_child_without_mother[dj])) + list(range(int(x_start_with_child_without_mother[dj]), + int(x_end_with_child_without_mother[dj]))) columns_covered_by_with_child_no_mothers = list(set(columns_covered_by_with_child_no_mothers)) all_columns = np.arange(len(peaks_neg_tot)-1) @@ -1970,8 +1970,8 @@ def return_boxes_of_images_by_order_of_reading_new( columns_covered_by_mothers = [] for dj in range(len(x_starting_all_between_nm_wc)): columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_starting_all_between_nm_wc[dj], - x_ending_all_between_nm_wc[dj])) + list(range(int(x_starting_all_between_nm_wc[dj]), + int(x_ending_all_between_nm_wc[dj]))) columns_covered_by_mothers = list(set(columns_covered_by_mothers)) all_columns=np.arange(i_s_nc, x_end_biggest_column) @@ -1979,8 +1979,8 @@ def return_boxes_of_images_by_order_of_reading_new( should_longest_line_be_extended=0 if (len(x_diff_all_between_nm_wc) > 0 and - set(list(range(x_starting_all_between_nm_wc[biggest], - x_ending_all_between_nm_wc[biggest])) + + set(list(range(int(x_starting_all_between_nm_wc[biggest]), + int(x_ending_all_between_nm_wc[biggest]))) + list(columns_not_covered)) != set(all_columns)): should_longest_line_be_extended=1 index_lines_so_close_to_top_separator = \ @@ -2012,7 +2012,7 @@ def return_boxes_of_images_by_order_of_reading_new( x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, np.array(columns_not_covered) + 1) ind_args_between=np.arange(len(x_ending_all_between_nm_wc)) - for column in range(i_s_nc, x_end_biggest_column): + for column in range(int(i_s_nc), int(x_end_biggest_column)): ind_args_in_col=ind_args_between[x_starting_all_between_nm_wc==column] #print('babali2') #print(ind_args_in_col,'ind_args_in_col') @@ -2064,7 +2064,7 @@ def return_boxes_of_images_by_order_of_reading_new( x_end_itself=x_end_copy.pop(il) #print(y_copy,'y_copy2') - for column in range(x_start_itself, x_end_itself+1): + for column in range(int(x_start_itself), int(x_end_itself)+1): #print(column,'cols') y_in_cols=[] for yic in range(len(y_copy)): @@ -2095,11 +2095,11 @@ def return_boxes_of_images_by_order_of_reading_new( all_columns = np.arange(len(peaks_neg_tot)-1) columns_covered_by_lines_covered_more_than_2col = [] for dj in range(len(x_starting)): - if set(list(range(x_starting[dj],x_ending[dj]))) == set(all_columns): + if set(list(range(int(x_starting[dj]),int(x_ending[dj]) ))) == set(all_columns): pass else: columns_covered_by_lines_covered_more_than_2col = columns_covered_by_lines_covered_more_than_2col + \ - list(range(x_starting[dj],x_ending[dj])) + list(range(int(x_starting[dj]),int(x_ending[dj]) )) columns_covered_by_lines_covered_more_than_2col = list(set(columns_covered_by_lines_covered_more_than_2col)) columns_not_covered = list(set(all_columns) - set(columns_covered_by_lines_covered_more_than_2col)) @@ -2124,7 +2124,7 @@ def return_boxes_of_images_by_order_of_reading_new( x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) ind_args=np.array(range(len(y_type_2))) - #ind_args=np.array(ind_args) + for column in range(len(peaks_neg_tot)-1): #print(column,'column') ind_args_in_col=ind_args[x_starting==column] @@ -2155,8 +2155,7 @@ def return_boxes_of_images_by_order_of_reading_new( x_start_itself=x_start_copy.pop(il) x_end_itself=x_end_copy.pop(il) - #print(y_copy,'y_copy2') - for column in range(x_start_itself, x_end_itself+1): + for column in range(int(x_start_itself), int(x_end_itself)+1): #print(column,'cols') y_in_cols=[] for yic in range(len(y_copy)): From 8ebba5ac046faff317e13455b94f79c6c510d782 Mon Sep 17 00:00:00 2001 From: michalbubula Date: Tue, 12 Aug 2025 16:21:15 +0200 Subject: [PATCH 178/374] add feedback to command line interface --- src/eynollah/eynollah.py | 305 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 290 insertions(+), 15 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index d47016b..d9939ca 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -203,6 +203,17 @@ class Eynollah: skip_layout_and_reading_order : bool = False, logger : Optional[Logger] = None, ): + if logger: + self.logger = logger + else: + self.logger = getLogger('eynollah') + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(message)s') + console_handler.setFormatter(formatter) + self.logger.addHandler(console_handler) + self.logger.setLevel(logging.INFO) + if skip_layout_and_reading_order: textline_light = True self.light_version = light_version @@ -237,10 +248,7 @@ class Eynollah: self.num_col_lower = int(num_col_lower) else: self.num_col_lower = num_col_lower - self.logger = logger if logger else getLogger('eynollah') - # for parallelization of CPU-intensive tasks: - self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) - atexit.register(self.executor.shutdown) + self.dir_models = dir_models self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" @@ -293,7 +301,14 @@ class Eynollah: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" else: self.model_table_dir = dir_models + "/eynollah-tables_20210319" + + + t_start = time.time() + # for parallelization of CPU-intensive tasks: + self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) + atexit.register(self.executor.shutdown) + # #gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) # #gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=7.7, allow_growth=True) # #session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) @@ -307,7 +322,11 @@ class Eynollah: tf.config.experimental.set_memory_growth(device, True) except: self.logger.warning("no GPU device available") - + + msg = "Loading models..." + print(msg) + self.logger.info(msg) + self.model_page = self.our_load_model(self.model_page_dir) self.model_classifier = self.our_load_model(self.model_dir_of_col_classifier) self.model_bin = self.our_load_model(self.model_dir_of_binarization) @@ -334,6 +353,10 @@ class Eynollah: self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") if self.tables: self.model_table = self.our_load_model(self.model_table_dir) + + msg = f"Model initialization complete ({time.time() - t_start:.1f}s)" + print(msg) + self.logger.info(msg) def cache_images(self, image_filename=None, image_pil=None, dpi=None): ret = {} @@ -4294,21 +4317,81 @@ class Eynollah: def run_single(self): t0 = time.time() + + msg = f"Processing file: {self.writer.image_filename}" + print(msg) + self.logger.info(msg) + + # Log enabled features directly + enabled_modes = [] + if self.light_version: + enabled_modes.append("Light version") + if self.textline_light: + enabled_modes.append("Light textline detection") + if self.full_layout: + enabled_modes.append("Full layout analysis") + if self.ocr: + enabled_modes.append("OCR") + if self.tables: + enabled_modes.append("Table detection") + + if enabled_modes: + msg = "Enabled modes: " + ", ".join(enabled_modes) + print(msg) + self.logger.info(msg) + + + msg = "Step 1/5: Image Enhancement" + print(msg) + self.logger.info(msg) + img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) - self.logger.info("Enhancing took %.1fs ", time.time() - t0) + + msg = f"Image: {self.image.shape[1]}x{self.image.shape[0]}, {self.dpi} DPI, {num_col_classifier} columns" + print(msg) + self.logger.info(msg) + if is_image_enhanced: + msg = "Enhancement applied" + print(msg) + self.logger.info(msg) + + msg = f"Enhancement complete ({time.time() - t0:.1f}s)" + print(msg) + self.logger.info(msg) + + + # Image Extraction Mode if self.extract_only_images: + msg = "Step 2/5: Image Extraction Mode" + print(msg) + self.logger.info(msg) + text_regions_p_1, erosion_hurts, polygons_lines_xml, polygons_of_images, image_page, page_coord, cont_page = \ self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) + ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], polygons_of_images, [], [], [], [], [], cont_page, [], [], ocr_all_textlines, []) + if self.plotter: self.plotter.write_images_into_directory(polygons_of_images, image_page) + + msg = "Image extraction complete" + print(msg) + self.logger.info(msg) return pcgts + # Basic Processing Mode if self.skip_layout_and_reading_order: + msg = "Step 2/5: Basic Processing Mode" + print(msg) + self.logger.info(msg) + msg = "Skipping layout analysis and reading order detection" + print(msg) + self.logger.info(msg) + _ ,_, _, textline_mask_tot_ea, img_bin_light, _ = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier, skip_layout_and_reading_order=self.skip_layout_and_reading_order) @@ -4349,11 +4432,21 @@ class Eynollah: all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + msg = "Basic processing complete" + print(msg) + self.logger.info(msg) return pcgts #print("text region early -1 in %.1fs", time.time() - t0) t1 = time.time() + msg = "Step 2/5: Layout Analysis" + print(msg) + self.logger.info(msg) + if self.light_version: + msg = "Using light version processing" + print(msg) + self.logger.info(msg) text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier) #print("text region early -2 in %.1fs", time.time() - t0) @@ -4384,20 +4477,30 @@ class Eynollah: text_regions_p_1 ,erosion_hurts, polygons_lines_xml = \ self.get_regions_from_xy_2models(img_res, is_image_enhanced, num_col_classifier) - self.logger.info("Textregion detection took %.1fs ", time.time() - t1) + msg = f"Textregion detection took {time.time() - t1:.1f}s" + print(msg) + self.logger.info(msg) confidence_matrix = np.zeros((text_regions_p_1.shape[:2])) t1 = time.time() num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ text_regions_p_1, cont_page, table_prediction = \ self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts) - self.logger.info("Graphics detection took %.1fs ", time.time() - t1) + msg = f"Graphics detection took {time.time() - t1:.1f}s" + print(msg) + self.logger.info(msg) #self.logger.info('cont_page %s', cont_page) #plt.imshow(table_prediction) #plt.show() + msg = f"Layout analysis complete ({time.time() - t1:.1f}s)" + print(msg) + self.logger.info(msg) if not num_col: - self.logger.info("No columns detected, outputting an empty PAGE-XML") + msg = "No columns detected - generating empty PAGE-XML" + print(msg) + self.logger.info(msg) + ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], [], [], [], [], [], [], @@ -4408,10 +4511,18 @@ class Eynollah: t1 = time.time() if not self.light_version: textline_mask_tot_ea = self.run_textline(image_page) - self.logger.info("textline detection took %.1fs", time.time() - t1) + msg = f"Textline detection took {time.time() - t1:.1f}s" + print(msg) + self.logger.info(msg) t1 = time.time() slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) - self.logger.info("deskewing took %.1fs", time.time() - t1) + if np.abs(slope_deskew) > 0.01: # Only log if there is significant skew + msg = f"Applied deskew correction: {slope_deskew:.2f} degrees" + print(msg) + self.logger.info(msg) + msg = f"Deskewing took {time.time() - t1:.1f}s" + print(msg) + self.logger.info(msg) elif num_col_classifier in (1,2): org_h_l_m = textline_mask_tot_ea.shape[0] org_w_l_m = textline_mask_tot_ea.shape[1] @@ -4431,6 +4542,19 @@ class Eynollah: textline_mask_tot, text_regions_p, image_page_rotated = \ self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction) + + msg = "Step 3/5: Text Line Detection" + print(msg) + self.logger.info(msg) + + if self.curved_line: + msg = "Mode: Curved line detection" + print(msg) + self.logger.info(msg) + elif self.textline_light: + msg = "Mode: Light detection" + print(msg) + self.logger.info(msg) if self.light_version and num_col_classifier in (1,2): image_page = resize_image(image_page,org_h_l_m, org_w_l_m ) @@ -4441,7 +4565,9 @@ class Eynollah: table_prediction = resize_image(table_prediction,org_h_l_m, org_w_l_m ) image_page_rotated = resize_image(image_page_rotated,org_h_l_m, org_w_l_m ) - self.logger.info("detection of marginals took %.1fs", time.time() - t1) + msg = f"Detection of marginals took {time.time() - t1:.1f}s" + print(msg) + self.logger.info(msg) #print("text region early 2 marginal in %.1fs", time.time() - t0) ## birdan sora chock chakir t1 = time.time() @@ -4540,7 +4666,9 @@ class Eynollah: cx_bigest_d_big[0] = cx_bigest_d[ind_largest] cy_biggest_d_big[0] = cy_biggest_d[ind_largest] except Exception as why: - self.logger.error(why) + msg = str(why) + print(f"Error: {msg}") + self.logger.error(msg) (h, w) = text_only.shape[:2] center = (w // 2.0, h // 2.0) @@ -4758,6 +4886,23 @@ class Eynollah: t_order = time.time() if self.full_layout: + msg = "Step 4/5: Reading Order Detection" + print(msg) + self.logger.info(msg) + + if self.reading_order_machine_based: + msg = "Using machine-based detection" + print(msg) + self.logger.info(msg) + if self.right2left: + msg = "Right-to-left mode enabled" + print(msg) + self.logger.info(msg) + if self.headers_off: + msg = "Headers ignored in reading order" + print(msg) + self.logger.info(msg) + if self.reading_order_machine_based: order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( contours_only_text_parent, contours_only_text_parent_h, text_regions_p) @@ -4768,21 +4913,84 @@ class Eynollah: else: order_text_new, id_of_texts_tot = self.do_order_of_regions( contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) - self.logger.info("detection of reading order took %.1fs", time.time() - t_order) + msg = f"Detection of reading order took {time.time() - t_order:.1f}s" + print(msg) + self.logger.info(msg) if self.ocr: + msg = "Step 4.5/5: OCR Processing" + print(msg) + self.logger.info(msg) + + if torch.cuda.is_available(): + msg = "Using GPU acceleration" + print(msg) + self.logger.info(msg) + else: + msg = "Using CPU processing" + print(msg) + self.logger.info(msg) + ocr_all_textlines = [] else: ocr_all_textlines = None + + msg = "Step 5/5: Output Generation" + print(msg) + self.logger.info(msg) + + output_config = [] + if self.enable_plotting: + output_config.append("Saving debug plots") + if self.dir_of_cropped_images: + output_config.append(f"Saving cropped images to: {self.dir_of_cropped_images}") + if self.dir_of_layout: + output_config.append(f"Saving layout plots to: {self.dir_of_layout}") + if self.dir_of_deskewed: + output_config.append(f"Saving deskewed images to: {self.dir_of_deskewed}") + + if output_config: + self.logger.info("Output configuration:\n * %s", "\n * ".join(output_config)) + pcgts = self.writer.build_pagexml_full_layout( contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_xml, ocr_all_textlines, conf_contours_textregions, conf_contours_textregions_h) + + summary = [ + f"Total processing time: {time.time() - t0:.1f}s", + f"Output file: {self.writer.output_filename}" + ] + + if self.ocr: + summary.append("OCR processing completed") + if self.full_layout: + summary.append("Full layout analysis completed") + if self.tables: + summary.append("Table detection completed") + return pcgts contours_only_text_parent_h = None + msg = "Step 4/5: Reading Order Detection" + print(msg) + self.logger.info(msg) + + if self.reading_order_machine_based: + msg = "Using machine-based detection" + print(msg) + self.logger.info(msg) + if self.right2left: + msg = "Right-to-left mode enabled" + print(msg) + self.logger.info(msg) + if self.headers_off: + msg = "Headers ignored in reading order" + print(msg) + self.logger.info(msg) + if self.reading_order_machine_based: order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( contours_only_text_parent, contours_only_text_parent_h, text_regions_p) @@ -4803,6 +5011,33 @@ class Eynollah: contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) if self.ocr: + msg = "Step 4.5/5: OCR Processing" + print(msg) + self.logger.info(msg) + + if torch.cuda.is_available(): + msg = "Using GPU acceleration" + print(msg) + self.logger.info(msg) + else: + msg = "Using CPU processing" + print(msg) + self.logger.info(msg) + + if self.light_version: + msg = "Using light version OCR" + print(msg) + self.logger.info(msg) + + if self.textline_light: + msg = "Using light text line detection for OCR" + print(msg) + self.logger.info(msg) + + msg = "Processing text lines..." + print(msg) + self.logger.info(msg) + device = cuda.get_current_device() device.reset() gc.collect() @@ -4853,12 +5088,52 @@ class Eynollah: else: ocr_all_textlines = None #print(ocr_all_textlines) - self.logger.info("detection of reading order took %.1fs", time.time() - t_order) + msg = f"Detection of reading order took {time.time() - t_order:.1f}s" + print(msg) + self.logger.info(msg) + + msg = "Step 5/5: Output Generation" + print(msg) + self.logger.info(msg) + + msg = "Generating PAGE-XML output" + print(msg) + self.logger.info(msg) + + if self.enable_plotting: + msg = "Saving debug plots" + print(msg) + self.logger.info(msg) + + if self.dir_of_cropped_images: + msg = f"Saving cropped images to: {self.dir_of_cropped_images}" + print(msg) + self.logger.info(msg) + + if self.dir_of_layout: + msg = f"Saving layout plots to: {self.dir_of_layout}" + print(msg) + self.logger.info(msg) + + if self.dir_of_deskewed: + msg = f"Saving deskewed images to: {self.dir_of_deskewed}" + print(msg) + self.logger.info(msg) + pcgts = self.writer.build_pagexml_no_full_layout( txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + + msg = f"\nProcessing completed in {time.time() - t0:.1f}s" + print(msg) + self.logger.info(msg) + + msg = f"Output file: {self.writer.output_filename}" + print(msg) + self.logger.info(msg) + return pcgts From 21615a986dbe2c6a2ddcf603b45ebe24e52f1e90 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 13 Aug 2025 14:14:37 +0200 Subject: [PATCH 179/374] OCR-D processor: expose reading_order_machine_based --- src/eynollah/ocrd-tool.json | 5 +++++ src/eynollah/processor.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index ce15206..af5e03f 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -74,6 +74,11 @@ "type": "boolean", "default": false, "description": "ignore the special role of headings during reading order detection" + }, + "reading_order_machine_based": { + "type": "boolean", + "default": false, + "description": "use data-driven (rather than rule-based) reading order detection" } }, "resources": [ diff --git a/src/eynollah/processor.py b/src/eynollah/processor.py index a53fede..c2922c1 100644 --- a/src/eynollah/processor.py +++ b/src/eynollah/processor.py @@ -24,6 +24,7 @@ class EynollahProcessor(Processor): allow_enhancement=self.parameter['allow_enhancement'], curved_line=self.parameter['curved_line'], right2left=self.parameter['right_to_left'], + reading_order_machine_based=self.parameter['reading_order_machine_based'], ignore_page_extraction=self.parameter['ignore_page_extraction'], light_version=self.parameter['light_version'], textline_light=self.parameter['textline_light'], @@ -57,6 +58,8 @@ class EynollahProcessor(Processor): - If ``ignore_page_extraction``, then attempt no cropping of the page. - If ``curved_line``, then compute contour polygons for text lines instead of simple bounding boxes. + - If ``reading_order_machine_based``, then detect reading order via + data-driven model instead of geometrical heuristics. Produce a new output file by serialising the resulting hierarchy. """ From 77415028769c55d48e6583ffc267e8d86a4a7cf0 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 18 Aug 2025 02:31:13 +0200 Subject: [PATCH 180/374] reading order on given layout --- src/eynollah/cli.py | 48 +- src/eynollah/mb_ro_on_layout.py | 1134 +++++++++++++++++++++++++++++++ 2 files changed, 1158 insertions(+), 24 deletions(-) create mode 100644 src/eynollah/mb_ro_on_layout.py diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 5135534..67fd57e 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -4,6 +4,7 @@ from ocrd_utils import initLogging, getLevelName, getLogger from eynollah.eynollah import Eynollah, Eynollah_ocr from eynollah.sbb_binarize import SbbBinarizer from eynollah.image_enhancer import Enhancer +from eynollah.mb_ro_on_layout import machine_based_reading_order_on_layout @click.group() def main(): @@ -13,38 +14,37 @@ def main(): @click.option( "--dir_xml", "-dx", - help="directory of GT page-xml files", + help="directory of page-xml files", type=click.Path(exists=True, file_okay=False), ) @click.option( - "--dir_out_modal_image", - "-domi", - help="directory where ground truth images would be written", + "--xml_file", + "-xml", + help="xml filename", + type=click.Path(exists=True, dir_okay=False), +) +@click.option( + "--dir_out", + "-do", + help="directory for output images", type=click.Path(exists=True, file_okay=False), ) @click.option( - "--dir_out_classes", - "-docl", - help="directory where ground truth classes would be written", + "--model", + "-m", + help="directory of models", type=click.Path(exists=True, file_okay=False), + required=True, ) -@click.option( - "--input_height", - "-ih", - help="input height", -) -@click.option( - "--input_width", - "-iw", - help="input width", -) -@click.option( - "--min_area_size", - "-min", - help="min area size of regions considered for reading order training.", -) -def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width, min_area_size): - xml_files_ind = os.listdir(dir_xml) + +def machine_based_reading_order(dir_xml, xml_file, dir_out, model): + raedingorder_object = machine_based_reading_order_on_layout(model, dir_out=dir_out, logger=getLogger('enhancement')) + + if dir_xml: + raedingorder_object.run(dir_in=dir_xml) + else: + raedingorder_object.run(xml_filename=xml_file) + @main.command() @click.option('--patches/--no-patches', default=True, help='by enabling this parameter you let the model to see the image in patches.') diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py new file mode 100644 index 0000000..7625a90 --- /dev/null +++ b/src/eynollah/mb_ro_on_layout.py @@ -0,0 +1,1134 @@ +""" +Image enhancer. The output can be written as same scale of input or in new predicted scale. +""" + +from logging import Logger +from difflib import SequenceMatcher as sq +from PIL import Image, ImageDraw, ImageFont +import math +import os +import sys +import time +from typing import Optional +import atexit +import warnings +from functools import partial +from pathlib import Path +from multiprocessing import cpu_count +import gc +import copy +from loky import ProcessPoolExecutor +import xml.etree.ElementTree as ET +import cv2 +import numpy as np +from ocrd import OcrdPage +from ocrd_utils import getLogger, tf_disable_interactive_logs +import statistics +from tensorflow.keras.models import load_model +from .utils.resize import resize_image +from .utils import ( + crop_image_inside_box +) + +from .utils.contour import ( + filter_contours_area_of_image, + filter_contours_area_of_image_tables, + find_contours_mean_y_diff, + find_new_features_of_contours, + find_features_of_contours, + get_text_region_boxes_by_given_contours, + get_textregion_contours_in_org_image, + get_textregion_contours_in_org_image_light, + return_contours_of_image, + return_contours_of_interested_region, + return_contours_of_interested_region_by_min_size, + return_contours_of_interested_textline, + return_parent_contours, +) + +DPI_THRESHOLD = 298 +KERNEL = np.ones((5, 5), np.uint8) + + +class machine_based_reading_order_on_layout: + def __init__( + self, + dir_models : str, + dir_out : Optional[str] = None, + logger : Optional[Logger] = None, + ): + self.dir_out = dir_out + + self.logger = logger if logger else getLogger('mbro on layout') + # for parallelization of CPU-intensive tasks: + self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) + atexit.register(self.executor.shutdown) + self.dir_models = dir_models + self.model_reading_order_dir = dir_models + "/model_step_5100000_mb_ro"#"/model_ens_reading_order_machine_based" + + try: + for device in tf.config.list_physical_devices('GPU'): + tf.config.experimental.set_memory_growth(device, True) + except: + self.logger.warning("no GPU device available") + + self.model_reading_order = self.our_load_model(self.model_reading_order_dir) + self.light_version = True + + + def cache_images(self, image_filename=None, image_pil=None, dpi=None): + ret = {} + t_c0 = time.time() + if image_filename: + ret['img'] = cv2.imread(image_filename) + if self.light_version: + self.dpi = 100 + else: + self.dpi = 0#check_dpi(image_filename) + else: + ret['img'] = pil2cv(image_pil) + if self.light_version: + self.dpi = 100 + else: + self.dpi = 0#check_dpi(image_pil) + ret['img_grayscale'] = cv2.cvtColor(ret['img'], cv2.COLOR_BGR2GRAY) + for prefix in ('', '_grayscale'): + ret[f'img{prefix}_uint8'] = ret[f'img{prefix}'].astype(np.uint8) + self._imgs = ret + if dpi is not None: + self.dpi = dpi + + def reset_file_name_dir(self, image_filename): + t_c = time.time() + self.cache_images(image_filename=image_filename) + self.output_filename = os.path.join(self.dir_out, Path(image_filename).stem +'.png') + + def imread(self, grayscale=False, uint8=True): + key = 'img' + if grayscale: + key += '_grayscale' + if uint8: + key += '_uint8' + return self._imgs[key].copy() + + def isNaN(self, num): + return num != num + + @staticmethod + def our_load_model(model_file): + if model_file.endswith('.h5') and Path(model_file[:-3]).exists(): + # prefer SavedModel over HDF5 format if it exists + model_file = model_file[:-3] + try: + model = load_model(model_file, compile=False) + except: + model = load_model(model_file, compile=False, custom_objects={ + "PatchEncoder": PatchEncoder, "Patches": Patches}) + return model + + def predict_enhancement(self, img): + self.logger.debug("enter predict_enhancement") + + img_height_model = self.model_enhancement.layers[-1].output_shape[1] + img_width_model = self.model_enhancement.layers[-1].output_shape[2] + if img.shape[0] < img_height_model: + img = cv2.resize(img, (img.shape[1], img_width_model), interpolation=cv2.INTER_NEAREST) + if img.shape[1] < img_width_model: + img = cv2.resize(img, (img_height_model, img.shape[0]), interpolation=cv2.INTER_NEAREST) + margin = int(0.1 * img_width_model) + width_mid = img_width_model - 2 * margin + height_mid = img_height_model - 2 * margin + img = img / 255. + img_h = img.shape[0] + img_w = img.shape[1] + + prediction_true = np.zeros((img_h, img_w, 3)) + nxf = img_w / float(width_mid) + nyf = img_h / float(height_mid) + nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) + nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + + for i in range(nxf): + for j in range(nyf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + else: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + if j == 0: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + else: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - img_width_model + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - img_height_model + + img_patch = img[np.newaxis, index_y_d:index_y_u, index_x_d:index_x_u, :] + label_p_pred = self.model_enhancement.predict(img_patch, verbose=0) + seg = label_p_pred[0, :, :, :] * 255 + + if i == 0 and j == 0: + prediction_true[index_y_d + 0:index_y_u - margin, + index_x_d + 0:index_x_u - margin] = \ + seg[0:-margin or None, + 0:-margin or None] + elif i == nxf - 1 and j == nyf - 1: + prediction_true[index_y_d + margin:index_y_u - 0, + index_x_d + margin:index_x_u - 0] = \ + seg[margin:, + margin:] + elif i == 0 and j == nyf - 1: + prediction_true[index_y_d + margin:index_y_u - 0, + index_x_d + 0:index_x_u - margin] = \ + seg[margin:, + 0:-margin or None] + elif i == nxf - 1 and j == 0: + prediction_true[index_y_d + 0:index_y_u - margin, + index_x_d + margin:index_x_u - 0] = \ + seg[0:-margin or None, + margin:] + elif i == 0 and j != 0 and j != nyf - 1: + prediction_true[index_y_d + margin:index_y_u - margin, + index_x_d + 0:index_x_u - margin] = \ + seg[margin:-margin or None, + 0:-margin or None] + elif i == nxf - 1 and j != 0 and j != nyf - 1: + prediction_true[index_y_d + margin:index_y_u - margin, + index_x_d + margin:index_x_u - 0] = \ + seg[margin:-margin or None, + margin:] + elif i != 0 and i != nxf - 1 and j == 0: + prediction_true[index_y_d + 0:index_y_u - margin, + index_x_d + margin:index_x_u - margin] = \ + seg[0:-margin or None, + margin:-margin or None] + elif i != 0 and i != nxf - 1 and j == nyf - 1: + prediction_true[index_y_d + margin:index_y_u - 0, + index_x_d + margin:index_x_u - margin] = \ + seg[margin:, + margin:-margin or None] + else: + prediction_true[index_y_d + margin:index_y_u - margin, + index_x_d + margin:index_x_u - margin] = \ + seg[margin:-margin or None, + margin:-margin or None] + + prediction_true = prediction_true.astype(int) + return prediction_true + + def calculate_width_height_by_columns(self, img, num_col, width_early, label_p_pred): + self.logger.debug("enter calculate_width_height_by_columns") + if num_col == 1: + img_w_new = 2000 + elif num_col == 2: + img_w_new = 2400 + elif num_col == 3: + img_w_new = 3000 + elif num_col == 4: + img_w_new = 4000 + elif num_col == 5: + img_w_new = 5000 + elif num_col == 6: + img_w_new = 6500 + else: + img_w_new = width_early + img_h_new = img_w_new * img.shape[0] // img.shape[1] + + if img_h_new >= 8000: + img_new = np.copy(img) + num_column_is_classified = False + else: + img_new = resize_image(img, img_h_new, img_w_new) + num_column_is_classified = True + + return img_new, num_column_is_classified + + def early_page_for_num_of_column_classification(self,img_bin): + self.logger.debug("enter early_page_for_num_of_column_classification") + if self.input_binary: + img = np.copy(img_bin).astype(np.uint8) + else: + img = self.imread() + img = cv2.GaussianBlur(img, (5, 5), 0) + img_page_prediction = self.do_prediction(False, img, self.model_page) + + imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + thresh = cv2.dilate(thresh, KERNEL, iterations=3) + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + if len(contours)>0: + cnt_size = np.array([cv2.contourArea(contours[j]) + for j in range(len(contours))]) + cnt = contours[np.argmax(cnt_size)] + box = cv2.boundingRect(cnt) + else: + box = [0, 0, img.shape[1], img.shape[0]] + cropped_page, page_coord = crop_image_inside_box(box, img) + + self.logger.debug("exit early_page_for_num_of_column_classification") + return cropped_page, page_coord + + def calculate_width_height_by_columns_1_2(self, img, num_col, width_early, label_p_pred): + self.logger.debug("enter calculate_width_height_by_columns") + if num_col == 1: + img_w_new = 1000 + else: + img_w_new = 1300 + img_h_new = img_w_new * img.shape[0] // img.shape[1] + + if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: + img_new = np.copy(img) + num_column_is_classified = False + #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: + elif img_h_new >= 8000: + img_new = np.copy(img) + num_column_is_classified = False + else: + img_new = resize_image(img, img_h_new, img_w_new) + num_column_is_classified = True + + return img_new, num_column_is_classified + + def resize_and_enhance_image_with_column_classifier(self, light_version): + self.logger.debug("enter resize_and_enhance_image_with_column_classifier") + dpi = 0#self.dpi + self.logger.info("Detected %s DPI", dpi) + if self.input_binary: + img = self.imread() + prediction_bin = self.do_prediction(True, img, self.model_bin, n_batch_inference=5) + prediction_bin = 255 * (prediction_bin[:,:,0]==0) + prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2).astype(np.uint8) + img= np.copy(prediction_bin) + img_bin = prediction_bin + else: + img = self.imread() + self.h_org, self.w_org = img.shape[:2] + img_bin = None + + width_early = img.shape[1] + t1 = time.time() + _, page_coord = self.early_page_for_num_of_column_classification(img_bin) + + self.image_page_org_size = img[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3], :] + self.page_coord = page_coord + + if self.num_col_upper and not self.num_col_lower: + num_col = self.num_col_upper + label_p_pred = [np.ones(6)] + elif self.num_col_lower and not self.num_col_upper: + num_col = self.num_col_lower + label_p_pred = [np.ones(6)] + elif not self.num_col_upper and not self.num_col_lower: + if self.input_binary: + img_in = np.copy(img) + img_in = img_in / 255.0 + img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = img_in.reshape(1, 448, 448, 3) + else: + img_1ch = self.imread(grayscale=True) + width_early = img_1ch.shape[1] + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + img_1ch = img_1ch / 255.0 + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = self.model_classifier.predict(img_in, verbose=0) + num_col = np.argmax(label_p_pred[0]) + 1 + elif (self.num_col_upper and self.num_col_lower) and (self.num_col_upper!=self.num_col_lower): + if self.input_binary: + img_in = np.copy(img) + img_in = img_in / 255.0 + img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = img_in.reshape(1, 448, 448, 3) + else: + img_1ch = self.imread(grayscale=True) + width_early = img_1ch.shape[1] + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + img_1ch = img_1ch / 255.0 + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = self.model_classifier.predict(img_in, verbose=0) + num_col = np.argmax(label_p_pred[0]) + 1 + + if num_col > self.num_col_upper: + num_col = self.num_col_upper + label_p_pred = [np.ones(6)] + if num_col < self.num_col_lower: + num_col = self.num_col_lower + label_p_pred = [np.ones(6)] + else: + num_col = self.num_col_upper + label_p_pred = [np.ones(6)] + + self.logger.info("Found %d columns (%s)", num_col, np.around(label_p_pred, decimals=5)) + + if dpi < DPI_THRESHOLD: + if light_version and num_col in (1,2): + img_new, num_column_is_classified = self.calculate_width_height_by_columns_1_2( + img, num_col, width_early, label_p_pred) + else: + img_new, num_column_is_classified = self.calculate_width_height_by_columns( + img, num_col, width_early, label_p_pred) + if light_version: + image_res = np.copy(img_new) + else: + image_res = self.predict_enhancement(img_new) + is_image_enhanced = True + + else: + num_column_is_classified = True + image_res = np.copy(img) + is_image_enhanced = False + + self.logger.debug("exit resize_and_enhance_image_with_column_classifier") + return is_image_enhanced, img, image_res, num_col, num_column_is_classified, img_bin + def read_xml(self, xml_file): + file_name = Path(xml_file).stem + tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) + root1=tree1.getroot() + alltags=[elem.tag for elem in root1.iter()] + link=alltags[0].split('}')[0]+'}' + + index_tot_regions = [] + tot_region_ref = [] + + for jj in root1.iter(link+'Page'): + y_len=int(jj.attrib['imageHeight']) + x_len=int(jj.attrib['imageWidth']) + + for jj in root1.iter(link+'RegionRefIndexed'): + index_tot_regions.append(jj.attrib['index']) + tot_region_ref.append(jj.attrib['regionRef']) + + if (link+'PrintSpace' in alltags) or (link+'Border' in alltags): + co_printspace = [] + if link+'PrintSpace' in alltags: + region_tags_printspace = np.unique([x for x in alltags if x.endswith('PrintSpace')]) + elif link+'Border' in alltags: + region_tags_printspace = np.unique([x for x in alltags if x.endswith('Border')]) + + for tag in region_tags_printspace: + if link+'PrintSpace' in alltags: + tag_endings_printspace = ['}PrintSpace','}printspace'] + elif link+'Border' in alltags: + tag_endings_printspace = ['}Border','}border'] + + if tag.endswith(tag_endings_printspace[0]) or tag.endswith(tag_endings_printspace[1]): + for nn in root1.iter(tag): + c_t_in = [] + sumi = 0 + for vv in nn.iter(): + # check the format of coords + if vv.tag == link + 'Coords': + coords = bool(vv.attrib) + if coords: + p_h = vv.attrib['points'].split(' ') + c_t_in.append( + np.array([[int(x.split(',')[0]), int(x.split(',')[1])] for x in p_h])) + break + else: + pass + + if vv.tag == link + 'Point': + c_t_in.append([int(float(vv.attrib['x'])), int(float(vv.attrib['y']))]) + sumi += 1 + elif vv.tag != link + 'Point' and sumi >= 1: + break + co_printspace.append(np.array(c_t_in)) + img_printspace = np.zeros( (y_len,x_len,3) ) + img_printspace=cv2.fillPoly(img_printspace, pts =co_printspace, color=(1,1,1)) + img_printspace = img_printspace.astype(np.uint8) + + imgray = cv2.cvtColor(img_printspace, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) + cnt = contours[np.argmax(cnt_size)] + x, y, w, h = cv2.boundingRect(cnt) + + bb_coord_printspace = [x, y, w, h] + + else: + bb_coord_printspace = None + + + region_tags=np.unique([x for x in alltags if x.endswith('Region')]) + co_text_paragraph=[] + co_text_drop=[] + co_text_heading=[] + co_text_header=[] + co_text_marginalia=[] + co_text_catch=[] + co_text_page_number=[] + co_text_signature_mark=[] + co_sep=[] + co_img=[] + co_table=[] + co_graphic=[] + co_graphic_text_annotation=[] + co_graphic_decoration=[] + co_noise=[] + + co_text_paragraph_text=[] + co_text_drop_text=[] + co_text_heading_text=[] + co_text_header_text=[] + co_text_marginalia_text=[] + co_text_catch_text=[] + co_text_page_number_text=[] + co_text_signature_mark_text=[] + co_sep_text=[] + co_img_text=[] + co_table_text=[] + co_graphic_text=[] + co_graphic_text_annotation_text=[] + co_graphic_decoration_text=[] + co_noise_text=[] + + id_paragraph = [] + id_header = [] + id_heading = [] + id_marginalia = [] + + for tag in region_tags: + if tag.endswith('}TextRegion') or tag.endswith('}Textregion'): + for nn in root1.iter(tag): + for child2 in nn: + tag2 = child2.tag + if tag2.endswith('}TextEquiv') or tag2.endswith('}TextEquiv'): + for childtext2 in child2: + if childtext2.tag.endswith('}Unicode') or childtext2.tag.endswith('}Unicode'): + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + co_text_drop_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='heading': + co_text_heading_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + co_text_signature_mark_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='header': + co_text_header_text.append(childtext2.text) + ###elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + ###co_text_catch_text.append(childtext2.text) + ###elif "type" in nn.attrib and nn.attrib['type']=='page-number': + ###co_text_page_number_text.append(childtext2.text) + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + co_text_marginalia_text.append(childtext2.text) + else: + co_text_paragraph_text.append(childtext2.text) + c_t_in_drop=[] + c_t_in_paragraph=[] + c_t_in_heading=[] + c_t_in_header=[] + c_t_in_page_number=[] + c_t_in_signature_mark=[] + c_t_in_catch=[] + c_t_in_marginalia=[] + + + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + + coords=bool(vv.attrib) + if coords: + #print('birda1') + p_h=vv.attrib['points'].split(' ') + + + + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + + c_t_in_drop.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='heading': + ##id_heading.append(nn.attrib['id']) + c_t_in_heading.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + + c_t_in_signature_mark.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + #print(c_t_in_paragraph) + elif "type" in nn.attrib and nn.attrib['type']=='header': + #id_header.append(nn.attrib['id']) + c_t_in_header.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + ###elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + ###c_t_in_catch.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + ###elif "type" in nn.attrib and nn.attrib['type']=='page-number': + + ###c_t_in_page_number.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + #id_marginalia.append(nn.attrib['id']) + + c_t_in_marginalia.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + else: + #id_paragraph.append(nn.attrib['id']) + + c_t_in_paragraph.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + break + else: + pass + + + if vv.tag==link+'Point': + if "type" in nn.attrib and nn.attrib['type']=='drop-capital': + + c_t_in_drop.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='heading': + #id_heading.append(nn.attrib['id']) + c_t_in_heading.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + + elif "type" in nn.attrib and nn.attrib['type']=='signature-mark': + + c_t_in_signature_mark.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + elif "type" in nn.attrib and nn.attrib['type']=='header': + #id_header.append(nn.attrib['id']) + c_t_in_header.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + + ###elif "type" in nn.attrib and nn.attrib['type']=='catch-word': + ###c_t_in_catch.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + ###sumi+=1 + + ###elif "type" in nn.attrib and nn.attrib['type']=='page-number': + + ###c_t_in_page_number.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + ###sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='marginalia': + #id_marginalia.append(nn.attrib['id']) + + c_t_in_marginalia.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + else: + #id_paragraph.append(nn.attrib['id']) + c_t_in_paragraph.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + + if len(c_t_in_drop)>0: + co_text_drop.append(np.array(c_t_in_drop)) + if len(c_t_in_paragraph)>0: + co_text_paragraph.append(np.array(c_t_in_paragraph)) + id_paragraph.append(nn.attrib['id']) + if len(c_t_in_heading)>0: + co_text_heading.append(np.array(c_t_in_heading)) + id_heading.append(nn.attrib['id']) + + if len(c_t_in_header)>0: + co_text_header.append(np.array(c_t_in_header)) + id_header.append(nn.attrib['id']) + if len(c_t_in_page_number)>0: + co_text_page_number.append(np.array(c_t_in_page_number)) + if len(c_t_in_catch)>0: + co_text_catch.append(np.array(c_t_in_catch)) + + if len(c_t_in_signature_mark)>0: + co_text_signature_mark.append(np.array(c_t_in_signature_mark)) + + if len(c_t_in_marginalia)>0: + co_text_marginalia.append(np.array(c_t_in_marginalia)) + id_marginalia.append(nn.attrib['id']) + + + elif tag.endswith('}GraphicRegion') or tag.endswith('}graphicregion'): + for nn in root1.iter(tag): + c_t_in=[] + c_t_in_text_annotation=[] + c_t_in_decoration=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + c_t_in_text_annotation.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + elif "type" in nn.attrib and nn.attrib['type']=='decoration': + c_t_in_decoration.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + else: + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + + + break + else: + pass + + + if vv.tag==link+'Point': + if "type" in nn.attrib and nn.attrib['type']=='handwritten-annotation': + c_t_in_text_annotation.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif "type" in nn.attrib and nn.attrib['type']=='decoration': + c_t_in_decoration.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + else: + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + if len(c_t_in_text_annotation)>0: + co_graphic_text_annotation.append(np.array(c_t_in_text_annotation)) + if len(c_t_in_decoration)>0: + co_graphic_decoration.append(np.array(c_t_in_decoration)) + if len(c_t_in)>0: + co_graphic.append(np.array(c_t_in)) + + + + elif tag.endswith('}ImageRegion') or tag.endswith('}imageregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + elif vv.tag!=link+'Point' and sumi>=1: + break + co_img.append(np.array(c_t_in)) + co_img_text.append(' ') + + + elif tag.endswith('}SeparatorRegion') or tag.endswith('}separatorregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + elif vv.tag!=link+'Point' and sumi>=1: + break + co_sep.append(np.array(c_t_in)) + + + + elif tag.endswith('}TableRegion') or tag.endswith('}tableregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_table.append(np.array(c_t_in)) + co_table_text.append(' ') + + elif tag.endswith('}NoiseRegion') or tag.endswith('}noiseregion'): + for nn in root1.iter(tag): + c_t_in=[] + sumi=0 + for vv in nn.iter(): + # check the format of coords + if vv.tag==link+'Coords': + coords=bool(vv.attrib) + if coords: + p_h=vv.attrib['points'].split(' ') + c_t_in.append( np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) ) + break + else: + pass + + + if vv.tag==link+'Point': + c_t_in.append([ int(float(vv.attrib['x'])) , int(float(vv.attrib['y'])) ]) + sumi+=1 + + elif vv.tag!=link+'Point' and sumi>=1: + break + co_noise.append(np.array(c_t_in)) + co_noise_text.append(' ') + + img = np.zeros( (y_len,x_len,3) ) + img_poly=cv2.fillPoly(img, pts =co_text_paragraph, color=(1,1,1)) + + img_poly=cv2.fillPoly(img, pts =co_text_heading, color=(2,2,2)) + img_poly=cv2.fillPoly(img, pts =co_text_header, color=(2,2,2)) + img_poly=cv2.fillPoly(img, pts =co_text_marginalia, color=(3,3,3)) + img_poly=cv2.fillPoly(img, pts =co_img, color=(4,4,4)) + img_poly=cv2.fillPoly(img, pts =co_sep, color=(5,5,5)) + + return tree1, root1, bb_coord_printspace, file_name, id_paragraph, id_header+id_heading, co_text_paragraph, co_text_header+co_text_heading,\ + tot_region_ref,x_len, y_len,index_tot_regions, img_poly + + def return_indexes_of_contours_loctaed_inside_another_list_of_contours(self, contours, contours_loc, cx_main_loc, cy_main_loc, indexes_loc): + indexes_of_located_cont = [] + center_x_coordinates_of_located = [] + center_y_coordinates_of_located = [] + #M_main_tot = [cv2.moments(contours_loc[j]) + #for j in range(len(contours_loc))] + #cx_main_loc = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] + #cy_main_loc = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] + + for ij in range(len(contours)): + results = [cv2.pointPolygonTest(contours[ij], (cx_main_loc[ind], cy_main_loc[ind]), False) + for ind in range(len(cy_main_loc)) ] + results = np.array(results) + indexes_in = np.where((results == 0) | (results == 1)) + indexes = indexes_loc[indexes_in]# [(results == 0) | (results == 1)]#np.where((results == 0) | (results == 1)) + + indexes_of_located_cont.append(indexes) + center_x_coordinates_of_located.append(np.array(cx_main_loc)[indexes_in] ) + center_y_coordinates_of_located.append(np.array(cy_main_loc)[indexes_in] ) + + return indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located + + def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p): + height1 =672#448 + width1 = 448#224 + + height2 =672#448 + width2= 448#224 + + height3 =672#448 + width3 = 448#224 + + inference_bs = 3 + + ver_kernel = np.ones((5, 1), dtype=np.uint8) + hor_kernel = np.ones((1, 5), dtype=np.uint8) + + + min_cont_size_to_be_dilated = 10 + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: + cx_conts, cy_conts, x_min_conts, x_max_conts, y_min_conts, y_max_conts, _ = find_new_features_of_contours(contours_only_text_parent) + args_cont_located = np.array(range(len(contours_only_text_parent))) + + diff_y_conts = np.abs(y_max_conts[:]-y_min_conts) + diff_x_conts = np.abs(x_max_conts[:]-x_min_conts) + + mean_x = statistics.mean(diff_x_conts) + median_x = statistics.median(diff_x_conts) + + + diff_x_ratio= diff_x_conts/mean_x + + args_cont_located_excluded = args_cont_located[diff_x_ratio>=1.3] + args_cont_located_included = args_cont_located[diff_x_ratio<1.3] + + contours_only_text_parent_excluded = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]>=1.3]#contours_only_text_parent[diff_x_ratio>=1.3] + contours_only_text_parent_included = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]<1.3]#contours_only_text_parent[diff_x_ratio<1.3] + + + cx_conts_excluded = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]>=1.3]#cx_conts[diff_x_ratio>=1.3] + cx_conts_included = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]<1.3]#cx_conts[diff_x_ratio<1.3] + + cy_conts_excluded = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]>=1.3]#cy_conts[diff_x_ratio>=1.3] + cy_conts_included = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]<1.3]#cy_conts[diff_x_ratio<1.3] + + #print(diff_x_ratio, 'ratio') + text_regions_p = text_regions_p.astype('uint8') + + if len(contours_only_text_parent_excluded)>0: + textregion_par = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1])).astype('uint8') + textregion_par = cv2.fillPoly(textregion_par, pts=contours_only_text_parent_included, color=(1,1)) + else: + textregion_par = (text_regions_p[:,:]==1)*1 + textregion_par = textregion_par.astype('uint8') + + text_regions_p_textregions_dilated = cv2.erode(textregion_par , hor_kernel, iterations=2) + text_regions_p_textregions_dilated = cv2.dilate(text_regions_p_textregions_dilated , ver_kernel, iterations=4) + text_regions_p_textregions_dilated = cv2.erode(text_regions_p_textregions_dilated , hor_kernel, iterations=1) + text_regions_p_textregions_dilated = cv2.dilate(text_regions_p_textregions_dilated , ver_kernel, iterations=5) + text_regions_p_textregions_dilated[text_regions_p[:,:]>1] = 0 + + + contours_only_dilated, hir_on_text_dilated = return_contours_of_image(text_regions_p_textregions_dilated) + contours_only_dilated = return_parent_contours(contours_only_dilated, hir_on_text_dilated) + + indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located = self.return_indexes_of_contours_loctaed_inside_another_list_of_contours(contours_only_dilated, contours_only_text_parent_included, cx_conts_included, cy_conts_included, args_cont_located_included) + + + if len(args_cont_located_excluded)>0: + for ind in args_cont_located_excluded: + indexes_of_located_cont.append(np.array([ind])) + contours_only_dilated.append(contours_only_text_parent[ind]) + center_y_coordinates_of_located.append(0) + + array_list = [np.array([elem]) if isinstance(elem, int) else elem for elem in indexes_of_located_cont] + flattened_array = np.concatenate([arr.ravel() for arr in array_list]) + #print(len( np.unique(flattened_array)), 'indexes_of_located_cont uniques') + + missing_textregions = list( set(np.array(range(len(contours_only_text_parent))) ) - set(np.unique(flattened_array)) ) + #print(missing_textregions, 'missing_textregions') + + for ind in missing_textregions: + indexes_of_located_cont.append(np.array([ind])) + contours_only_dilated.append(contours_only_text_parent[ind]) + center_y_coordinates_of_located.append(0) + + + if contours_only_text_parent_h: + for vi in range(len(contours_only_text_parent_h)): + indexes_of_located_cont.append(int(vi+len(contours_only_text_parent))) + + array_list = [np.array([elem]) if isinstance(elem, int) else elem for elem in indexes_of_located_cont] + flattened_array = np.concatenate([arr.ravel() for arr in array_list]) + + y_len = text_regions_p.shape[0] + x_len = text_regions_p.shape[1] + + img_poly = np.zeros((y_len,x_len), dtype='uint8') + img_poly[text_regions_p[:,:]==1] = 1 + img_poly[text_regions_p[:,:]==2] = 2 + img_poly[text_regions_p[:,:]==3] = 4 + img_poly[text_regions_p[:,:]==6] = 5 + + img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') + if contours_only_text_parent_h: + _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours( + contours_only_text_parent_h) + for j in range(len(cy_main)): + img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12, + int(x_min_main[j]):int(x_max_main[j])] = 1 + co_text_all_org = contours_only_text_parent + contours_only_text_parent_h + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: + co_text_all = contours_only_dilated + contours_only_text_parent_h + else: + co_text_all = contours_only_text_parent + contours_only_text_parent_h + else: + co_text_all_org = contours_only_text_parent + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: + co_text_all = contours_only_dilated + else: + co_text_all = contours_only_text_parent + + if not len(co_text_all): + return [], [] + + labels_con = np.zeros((int(y_len /6.), int(x_len/6.), len(co_text_all)), dtype=bool) + + co_text_all = [(i/6).astype(int) for i in co_text_all] + for i in range(len(co_text_all)): + img = labels_con[:,:,i].astype(np.uint8) + + #img = cv2.resize(img, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) + + cv2.fillPoly(img, pts=[co_text_all[i]], color=(1,)) + labels_con[:,:,i] = img + + + labels_con = resize_image(labels_con.astype(np.uint8), height1, width1).astype(bool) + img_header_and_sep = resize_image(img_header_and_sep, height1, width1) + img_poly = resize_image(img_poly, height3, width3) + + + + input_1 = np.zeros((inference_bs, height1, width1, 3)) + ordered = [list(range(len(co_text_all)))] + index_update = 0 + #print(labels_con.shape[2],"number of regions for reading order") + while index_update>=0: + ij_list = ordered.pop(index_update) + i = ij_list.pop(0) + + ante_list = [] + post_list = [] + tot_counter = 0 + batch = [] + for j in ij_list: + img1 = labels_con[:,:,i].astype(float) + img2 = labels_con[:,:,j].astype(float) + img1[img_poly==5] = 2 + img2[img_poly==5] = 2 + img1[img_header_and_sep==1] = 3 + img2[img_header_and_sep==1] = 3 + + input_1[len(batch), :, :, 0] = img1 / 3. + input_1[len(batch), :, :, 2] = img2 / 3. + input_1[len(batch), :, :, 1] = img_poly / 5. + + tot_counter += 1 + batch.append(j) + if tot_counter % inference_bs == 0 or tot_counter == len(ij_list): + y_pr = self.model_reading_order.predict(input_1 , verbose=0) + for jb, j in enumerate(batch): + if y_pr[jb][0]>=0.5: + post_list.append(j) + else: + ante_list.append(j) + batch = [] + + if len(ante_list): + ordered.insert(index_update, ante_list) + index_update += 1 + ordered.insert(index_update, [i]) + if len(post_list): + ordered.insert(index_update + 1, post_list) + + index_update = -1 + for index_next, ij_list in enumerate(ordered): + if len(ij_list) > 1: + index_update = index_next + break + + ordered = [i[0] for i in ordered] + + ##id_all_text = np.array(id_all_text)[index_sort] + + + if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: + org_contours_indexes = [] + for ind in range(len(ordered)): + region_with_curr_order = ordered[ind] + if region_with_curr_order < len(contours_only_dilated): + if np.isscalar(indexes_of_located_cont[region_with_curr_order]): + org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]] + else: + arg_sort_located_cont = np.argsort(center_y_coordinates_of_located[region_with_curr_order]) + org_contours_indexes = org_contours_indexes + list(np.array(indexes_of_located_cont[region_with_curr_order])[arg_sort_located_cont]) ##org_contours_indexes + list ( + else: + org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]] + + region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] + return org_contours_indexes, region_ids + else: + region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] + return ordered, region_ids + + + + + def run(self, xml_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): + """ + Get image and scales, then extract the page of scanned image + """ + self.logger.debug("enter run") + t0_tot = time.time() + + if dir_in: + self.ls_xmls = os.listdir(dir_in) + elif xml_filename: + self.ls_xmls = [xml_filename] + else: + raise ValueError("run requires either a single image filename or a directory") + + for xml_filename in self.ls_xmls: + self.logger.info(xml_filename) + t0 = time.time() + + if dir_in: + xml_file = os.path.join(dir_in, xml_filename) + else: + xml_file = xml_filename + + tree_xml, root_xml, bb_coord_printspace, file_name, id_paragraph, id_header, co_text_paragraph, co_text_header, tot_region_ref, x_len, y_len, index_tot_regions, img_poly = self.read_xml(xml_file) + + id_all_text = id_paragraph + id_header + + order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model(co_text_paragraph, co_text_header, img_poly[:,:,0]) + + id_all_text = np.array(id_all_text)[order_text_new] + + alltags=[elem.tag for elem in root_xml.iter()] + + + + link=alltags[0].split('}')[0]+'}' + name_space = alltags[0].split('}')[0] + name_space = name_space.split('{')[1] + + page_element = root_xml.find(link+'Page') + + + old_ro = root_xml.find(".//{*}ReadingOrder") + + if old_ro is not None: + page_element.remove(old_ro) + + #print(old_ro, 'old_ro') + ro_subelement = ET.Element('ReadingOrder') + + ro_subelement2 = ET.SubElement(ro_subelement, 'OrderedGroup') + ro_subelement2.set('id', "ro357564684568544579089") + + for index, id_text in enumerate(id_all_text): + new_element_2 = ET.SubElement(ro_subelement2, 'RegionRefIndexed') + new_element_2.set('regionRef', id_all_text[index]) + new_element_2.set('index', str(index)) + + if (link+'PrintSpace' in alltags) or (link+'Border' in alltags): + page_element.insert(1, ro_subelement) + else: + page_element.insert(0, ro_subelement) + + alltags=[elem.tag for elem in root_xml.iter()] + + ET.register_namespace("",name_space) + tree_xml.write(os.path.join(self.dir_out, file_name+'.xml'),xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) + + #sys.exit() + From 41365645efd7690ace773a78e4334b31090f055c Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 26 Aug 2025 22:38:03 +0200 Subject: [PATCH 181/374] Marginals are divided into left and right, and written from top to bottom. --- src/eynollah/eynollah.py | 138 ++++++++++++++++++++++++-------- src/eynollah/mb_ro_on_layout.py | 18 +++-- src/eynollah/utils/utils_ocr.py | 88 ++++++++++---------- src/eynollah/utils/xml.py | 10 ++- src/eynollah/writer.py | 58 ++++++++++---- 5 files changed, 215 insertions(+), 97 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 5299d3e..30e180d 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -289,7 +289,7 @@ class Eynollah: self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_step_4800000_mb_ro"#"/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_eynollah_reading_order_20250824"#"/model_mb_ro_aug_ens_11"#"/model_step_3200000_mb_ro"#"/model_ens_reading_order_machine_based"#"/model_mb_ro_aug_ens_8"#"/model_ens_reading_order_machine_based" #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" @@ -725,6 +725,7 @@ class Eynollah: label_p_pred = self.model_classifier.predict(img_in, verbose=0) num_col = np.argmax(label_p_pred[0]) + 1 + elif (self.num_col_upper and self.num_col_lower) and (self.num_col_upper!=self.num_col_lower): if self.input_binary: img_in = np.copy(img) @@ -3090,6 +3091,26 @@ class Eynollah: num_col = num_col + 1 if not num_column_is_classified: num_col_classifier = num_col + 1 + if self.num_col_upper and self.num_col_lower: + if self.num_col_upper == self.num_col_lower: + num_col_classifier = self.num_col_upper + else: + if num_col_classifier < self.num_col_lower: + num_col_classifier = self.num_col_lower + if num_col_classifier > self.num_col_upper: + num_col_classifier = self.num_col_upper + + elif self.num_col_lower and not self.num_col_upper: + if num_col_classifier < self.num_col_lower: + num_col_classifier = self.num_col_lower + + elif self.num_col_upper and not self.num_col_lower: + if num_col_classifier > self.num_col_upper: + num_col_classifier = self.num_col_upper + + else: + pass + except Exception as why: self.logger.error(why) num_col = None @@ -3223,7 +3244,6 @@ class Eynollah: text_regions_p_1[mask_lines[:, :] == 1] = 3 text_regions_p = text_regions_p_1[:, :] text_regions_p = np.array(text_regions_p) - if num_col_classifier in (1, 2): try: regions_without_separators = (text_regions_p[:, :] == 1) * 1 @@ -4447,6 +4467,43 @@ class Eynollah: return (slopes_rem, all_found_textline_polygons_rem, boxes_text_rem, txt_con_org_rem, contours_only_text_parent_rem, index_by_text_par_con_rem_sort) + + def separate_marginals_to_left_and_right_and_order_from_top_to_down(self, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes_marginals, mid_point_of_page_width): + cx_marg, cy_marg, _, _, _, _, _ = find_new_features_of_contours( + polygons_of_marginals) + + cx_marg = np.array(cx_marg) + cy_marg = np.array(cy_marg) + + poly_marg_left = list( np.array(polygons_of_marginals)[cx_marg < mid_point_of_page_width] ) + poly_marg_right = list( np.array(polygons_of_marginals)[cx_marg >= mid_point_of_page_width] ) + + all_found_textline_polygons_marginals_left = list( np.array(all_found_textline_polygons_marginals)[cx_marg < mid_point_of_page_width] ) + all_found_textline_polygons_marginals_right = list( np.array(all_found_textline_polygons_marginals)[cx_marg >= mid_point_of_page_width] ) + + all_box_coord_marginals_left = list( np.array(all_box_coord_marginals)[cx_marg < mid_point_of_page_width] ) + all_box_coord_marginals_right = list( np.array(all_box_coord_marginals)[cx_marg >= mid_point_of_page_width] ) + + slopes_marg_left = list( np.array(slopes_marginals)[cx_marg < mid_point_of_page_width] ) + slopes_marg_right = list( np.array(slopes_marginals)[cx_marg >= mid_point_of_page_width] ) + + cy_marg_left = cy_marg[cx_marg < mid_point_of_page_width] + cy_marg_right = cy_marg[cx_marg >= mid_point_of_page_width] + + ordered_left_marginals = [poly for _, poly in sorted(zip(cy_marg_left, poly_marg_left), key=lambda x: x[0])] + ordered_right_marginals = [poly for _, poly in sorted(zip(cy_marg_right, poly_marg_right), key=lambda x: x[0])] + + ordered_left_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_left, all_found_textline_polygons_marginals_left), key=lambda x: x[0])] + ordered_right_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_right, all_found_textline_polygons_marginals_right), key=lambda x: x[0])] + + ordered_left_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_left, all_box_coord_marginals_left), key=lambda x: x[0])] + ordered_right_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_right, all_box_coord_marginals_right), key=lambda x: x[0])] + + ordered_left_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_left, slopes_marg_left), key=lambda x: x[0])] + ordered_right_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_right, slopes_marg_right), key=lambda x: x[0])] + + return ordered_left_marginals, ordered_right_marginals, ordered_left_marginals_textline, ordered_right_marginals_textline, ordered_left_marginals_bbox, ordered_right_marginals_bbox, ordered_left_slopes_marginals, ordered_right_slopes_marginals + def run(self, image_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): """ @@ -4489,12 +4546,13 @@ class Eynollah: t0 = time.time() img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) self.logger.info("Enhancing took %.1fs ", time.time() - t0) + if self.extract_only_images: text_regions_p_1, erosion_hurts, polygons_lines_xml, polygons_of_images, image_page, page_coord, cont_page = \ self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], - polygons_of_images, [], [], [], [], [], + polygons_of_images, [], [], [], [], [], [], [], [], [], cont_page, [], []) if self.plotter: self.plotter.write_images_into_directory(polygons_of_images, image_page) @@ -4508,7 +4566,6 @@ class Eynollah: page_coord, image_page, textline_mask_tot_ea, img_bin_light, cont_page = \ self.run_graphics_and_columns_without_layout(textline_mask_tot_ea, img_bin_light) - ##all_found_textline_polygons =self.scale_contours_new(textline_mask_tot_ea) cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(textline_mask_tot_ea) @@ -4530,10 +4587,14 @@ class Eynollah: id_of_texts_tot =['region_0001'] polygons_of_images = [] - slopes_marginals = [] - polygons_of_marginals = [] - all_found_textline_polygons_marginals = [] - all_box_coord_marginals = [] + slopes_marginals_left = [] + slopes_marginals_right = [] + polygons_of_marginals_left = [] + polygons_of_marginals_right = [] + all_found_textline_polygons_marginals_left = [] + all_found_textline_polygons_marginals_right = [] + all_box_coord_marginals_left = [] + all_box_coord_marginals_right = [] polygons_lines_xml = [] contours_tables = [] conf_contours_textregions =[0] @@ -4546,8 +4607,8 @@ class Eynollah: pcgts = self.writer.build_pagexml_no_full_layout( cont_page, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, - all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, + all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines=ocr_all_textlines, conf_contours_textregion=conf_contours_textregions, skip_layout_reading_order=self.skip_layout_and_reading_order) return pcgts @@ -4595,11 +4656,10 @@ class Eynollah: #self.logger.info('cont_page %s', cont_page) #plt.imshow(table_prediction) #plt.show() - if not num_col: self.logger.info("No columns detected, outputting an empty PAGE-XML") pcgts = self.writer.build_pagexml_no_full_layout( - [], page_coord, [], [], [], [], [], [], [], [], [], [], + [], page_coord, [], [], [], [], [], [], [], [], [], [], [], [], [], [], cont_page, [], []) return pcgts @@ -4771,6 +4831,7 @@ class Eynollah: contours_only_text_parent_d_ordered = [] contours_only_text_parent_d = [] #contours_only_text_parent = [] + if not len(contours_only_text_parent): # stop early empty_marginals = [[]] * len(polygons_of_marginals) @@ -4778,13 +4839,13 @@ class Eynollah: pcgts = self.writer.build_pagexml_full_layout( [], [], page_coord, [], [], [], [], [], [], polygons_of_images, contours_tables, [], - polygons_of_marginals, empty_marginals, empty_marginals, [], [], [], + polygons_of_marginals, polygons_of_marginals, empty_marginals, empty_marginals, empty_marginals, empty_marginals, [], [], [], [], cont_page, polygons_lines_xml) else: pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], polygons_of_images, - polygons_of_marginals, empty_marginals, empty_marginals, [], [], + polygons_of_marginals, polygons_of_marginals, empty_marginals, empty_marginals, empty_marginals, empty_marginals, [], [], [], cont_page, polygons_lines_xml, contours_tables) return pcgts @@ -4877,8 +4938,11 @@ class Eynollah: num_col_classifier, scale_param, slope_deskew) all_found_textline_polygons_marginals = small_textlines_to_parent_adherence2( all_found_textline_polygons_marginals, textline_mask_tot_ea, num_col_classifier) - - #print("text region early 6 in %.1fs", time.time() - t0) + + mid_point_of_page_width = text_regions_p.shape[1] / 2. + polygons_of_marginals_left, polygons_of_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes_marginals_left, slopes_marginals_right = self.separate_marginals_to_left_and_right_and_order_from_top_to_down(polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes_marginals, mid_point_of_page_width) + + #print(len(polygons_of_marginals), len(ordered_left_marginals), len(ordered_right_marginals), 'marginals ordred') if self.full_layout: if np.abs(slope_deskew) >= SLOPE_THRESHOLD: contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( @@ -4961,7 +5025,6 @@ class Eynollah: tror = time.time() order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( contours_only_text_parent, contours_only_text_parent_h, text_regions_p) - print('time spend for mb ro', time.time()-tror) else: if np.abs(slope_deskew) < SLOPE_THRESHOLD: order_text_new, id_of_texts_tot = self.do_order_of_regions( @@ -4978,10 +5041,15 @@ class Eynollah: else: ocr_all_textlines = None - if all_found_textline_polygons_marginals and len(all_found_textline_polygons_marginals)>0: - ocr_all_textlines_marginals = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: + ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_left, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: - ocr_all_textlines_marginals = None + ocr_all_textlines_marginals_left = None + + if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: + ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_right, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + else: + ocr_all_textlines_marginals_right = None if all_found_textline_polygons_h and len(all_found_textline_polygons)>0: ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_h, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) @@ -4994,15 +5062,16 @@ class Eynollah: ocr_all_textlines_drop = None else: ocr_all_textlines = None - ocr_all_textlines_marginals = None + ocr_all_textlines_marginals_left = None + ocr_all_textlines_marginals_right = None ocr_all_textlines_h = None ocr_all_textlines_drop = None pcgts = self.writer.build_pagexml_full_layout( contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, - polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals, - all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, - cont_page, polygons_lines_xml, ocr_all_textlines, ocr_all_textlines_h, ocr_all_textlines_marginals, ocr_all_textlines_drop, conf_contours_textregions, conf_contours_textregions_h) + polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_lines_xml, ocr_all_textlines, ocr_all_textlines_h, ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, ocr_all_textlines_drop, conf_contours_textregions, conf_contours_textregions_h) return pcgts contours_only_text_parent_h = None @@ -5077,19 +5146,24 @@ class Eynollah: gc.collect() if len(all_found_textline_polygons)>0: ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - if all_found_textline_polygons_marginals and len(all_found_textline_polygons_marginals)>0: - ocr_all_textlines_marginals = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + + if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: + ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_left, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + + if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: + ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_right, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines = None - ocr_all_textlines_marginals = None + ocr_all_textlines_marginals_left = None + ocr_all_textlines_marginals_right = None self.logger.info("detection of reading order took %.1fs", time.time() - t_order) pcgts = self.writer.build_pagexml_no_full_layout( txt_con_org, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, - all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, ocr_all_textlines_marginals, conf_contours_textregions) + all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, conf_contours_textregions) return pcgts @@ -5145,7 +5219,7 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250805" + self.model_ocr_dir = dir_models + "/model_step_45000_ocr"#"/model_eynollah_ocr_cnnrnn_20250805"# model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5156,7 +5230,7 @@ class Eynollah_ocr: else: self.b_s = int(batch_size) - with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: + with open(os.path.join(self.model_ocr_dir, "characters_20250707_all_lang.txt"),"r") as config_file: characters = json.load(config_file) AUTOTUNE = tf.data.AUTOTUNE diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index 7625a90..c03d831 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -64,7 +64,7 @@ class machine_based_reading_order_on_layout: self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) atexit.register(self.executor.shutdown) self.dir_models = dir_models - self.model_reading_order_dir = dir_models + "/model_step_5100000_mb_ro"#"/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_eynollah_reading_order_20250824"#"/model_ens_reading_order_machine_based" try: for device in tf.config.list_physical_devices('GPU'): @@ -942,10 +942,18 @@ class machine_based_reading_order_on_layout: x_len = text_regions_p.shape[1] img_poly = np.zeros((y_len,x_len), dtype='uint8') - img_poly[text_regions_p[:,:]==1] = 1 - img_poly[text_regions_p[:,:]==2] = 2 - img_poly[text_regions_p[:,:]==3] = 4 - img_poly[text_regions_p[:,:]==6] = 5 + ###img_poly[text_regions_p[:,:]==1] = 1 + ###img_poly[text_regions_p[:,:]==2] = 2 + ###img_poly[text_regions_p[:,:]==3] = 4 + ###img_poly[text_regions_p[:,:]==6] = 5 + + ##img_poly[text_regions_p[:,:]==1] = 1 + ##img_poly[text_regions_p[:,:]==2] = 2 + ##img_poly[text_regions_p[:,:]==3] = 3 + ##img_poly[text_regions_p[:,:]==4] = 4 + ##img_poly[text_regions_p[:,:]==5] = 5 + + img_poly = np.copy(text_regions_p) img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') if contours_only_text_parent_h: diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 1e9162a..d974650 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -384,57 +384,63 @@ def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, pr for indexing, ind_poly_first in enumerate(all_found_textline_polygons): #ocr_textline_in_textregion = [] - for indexing2, ind_poly in enumerate(ind_poly_first): + if len(ind_poly_first)==0: cropped_lines_region_indexer.append(indexer_text_region) - if not (textline_light or curved_line): - ind_poly = copy.deepcopy(ind_poly) - box_ind = all_box_coord[indexing] + cropped_lines_meging_indexing.append(0) + img_fin = np.ones((image_height, image_width, 3))*1 + cropped_lines.append(img_fin) - ind_poly = return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) - #print(ind_poly_copy) - ind_poly[ind_poly<0] = 0 - x, y, w, h = cv2.boundingRect(ind_poly) - - w_scaled = w * image_height/float(h) + else: + for indexing2, ind_poly in enumerate(ind_poly_first): + cropped_lines_region_indexer.append(indexer_text_region) + if not (textline_light or curved_line): + ind_poly = copy.deepcopy(ind_poly) + box_ind = all_box_coord[indexing] - mask_poly = np.zeros(image.shape) - - img_poly_on_img = np.copy(image) - - mask_poly = cv2.fillPoly(mask_poly, pts=[ind_poly], color=(1, 1, 1)) - - - - mask_poly = mask_poly[y:y+h, x:x+w, :] - img_crop = img_poly_on_img[y:y+h, x:x+w, :] - - img_crop[mask_poly==0] = 255 - - if w_scaled < 640:#1.5*image_width: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(0) - else: - splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) + ind_poly = return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) + #print(ind_poly_copy) + ind_poly[ind_poly<0] = 0 + x, y, w, h = cv2.boundingRect(ind_poly) - if splited_images: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(1) - - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) - - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(-1) - - else: + w_scaled = w * image_height/float(h) + + mask_poly = np.zeros(image.shape) + + img_poly_on_img = np.copy(image) + + mask_poly = cv2.fillPoly(mask_poly, pts=[ind_poly], color=(1, 1, 1)) + + + + mask_poly = mask_poly[y:y+h, x:x+w, :] + img_crop = img_poly_on_img[y:y+h, x:x+w, :] + + img_crop[mask_poly==0] = 255 + + if w_scaled < 640:#1.5*image_width: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) + else: + splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) + + if splited_images: + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(1) + + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) + + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(-1) + + else: + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + cropped_lines.append(img_fin) + cropped_lines_meging_indexing.append(0) indexer_text_region+=1 - extracted_texts = [] n_iterations = math.ceil(len(cropped_lines) / b_s_ocr) diff --git a/src/eynollah/utils/xml.py b/src/eynollah/utils/xml.py index bd95702..13420df 100644 --- a/src/eynollah/utils/xml.py +++ b/src/eynollah/utils/xml.py @@ -46,16 +46,22 @@ def create_page_xml(imageFilename, height, width): )) return pcgts -def xml_reading_order(page, order_of_texts, id_of_marginalia): +def xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right): region_order = ReadingOrderType() og = OrderedGroupType(id="ro357564684568544579089") page.set_ReadingOrder(region_order) region_order.set_OrderedGroup(og) region_counter = EynollahIdCounter() + + for id_marginal in id_of_marginalia_left: + og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=id_marginal)) + region_counter.inc('region') + for idx_textregion, _ in enumerate(order_of_texts): og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=region_counter.region_id(order_of_texts[idx_textregion] + 1))) region_counter.inc('region') - for id_marginal in id_of_marginalia: + + for id_marginal in id_of_marginalia_right: og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=id_marginal)) region_counter.inc('region') diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 085ee6f..2f9caf3 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -170,7 +170,7 @@ class EynollahXmlWriter(): with open(self.output_filename, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines=None, ocr_all_textlines_marginals=None, conf_contours_textregion=None, skip_layout_reading_order=False): + def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals_left, found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, conf_contours_textregion=None, skip_layout_reading_order=False): self.logger.debug('enter build_pagexml_no_full_layout') # create the file structure @@ -181,8 +181,9 @@ class EynollahXmlWriter(): counter = EynollahIdCounter() if len(found_polygons_text_region) > 0: _counter_marginals = EynollahIdCounter(region_idx=len(order_of_texts)) - id_of_marginalia = [_counter_marginals.next_region_id for _ in found_polygons_marginals] - xml_reading_order(page, order_of_texts, id_of_marginalia) + id_of_marginalia_left = [_counter_marginals.next_region_id for _ in found_polygons_marginals_left] + id_of_marginalia_right = [_counter_marginals.next_region_id for _ in found_polygons_marginals_right] + xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right) for mm in range(len(found_polygons_text_region)): textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', @@ -195,17 +196,29 @@ class EynollahXmlWriter(): else: ocr_textlines = None self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, all_box_coord, slopes, counter, ocr_textlines) - - for mm in range(len(found_polygons_marginals)): + + for mm in range(len(found_polygons_marginals_left)): marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_left[mm], page_coord))) page.add_TextRegion(marginal) - if ocr_all_textlines_marginals: - ocr_textlines = ocr_all_textlines_marginals[mm] + if ocr_all_textlines_marginals_left: + ocr_textlines = ocr_all_textlines_marginals_left[mm] else: ocr_textlines = None - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter, ocr_textlines) + #print(ocr_textlines, mm, len(all_found_textline_polygons_marginals_left[mm]) ) + self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) + + for mm in range(len(found_polygons_marginals_right)): + marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_right[mm], page_coord))) + page.add_TextRegion(marginal) + if ocr_all_textlines_marginals_right: + ocr_textlines = ocr_all_textlines_marginals_right[mm] + else: + ocr_textlines = None + + self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) for mm in range(len(found_polygons_text_region_img)): img_region = ImageRegionType(id=counter.next_region_id, Coords=CoordsType()) @@ -249,7 +262,7 @@ class EynollahXmlWriter(): return pcgts - def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals=None, ocr_all_textlines_drop=None, conf_contours_textregion=None, conf_contours_textregion_h=None): + def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals_left,found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_lines_to_be_written_in_xml, ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, ocr_all_textlines_drop=None, conf_contours_textregion=None, conf_contours_textregion_h=None): self.logger.debug('enter build_pagexml_full_layout') # create the file structure @@ -259,8 +272,9 @@ class EynollahXmlWriter(): counter = EynollahIdCounter() _counter_marginals = EynollahIdCounter(region_idx=len(order_of_texts)) - id_of_marginalia = [_counter_marginals.next_region_id for _ in found_polygons_marginals] - xml_reading_order(page, order_of_texts, id_of_marginalia) + id_of_marginalia_left = [_counter_marginals.next_region_id for _ in found_polygons_marginals_left] + id_of_marginalia_right = [_counter_marginals.next_region_id for _ in found_polygons_marginals_right] + xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right) for mm in range(len(found_polygons_text_region)): textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', @@ -285,15 +299,25 @@ class EynollahXmlWriter(): ocr_textlines = None self.serialize_lines_in_region(textregion, all_found_textline_polygons_h, mm, page_coord, all_box_coord_h, slopes_h, counter, ocr_textlines) - for mm in range(len(found_polygons_marginals)): + for mm in range(len(found_polygons_marginals_left)): marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_left[mm], page_coord))) page.add_TextRegion(marginal) - if ocr_all_textlines_marginals: - ocr_textlines = ocr_all_textlines_marginals[mm] + if ocr_all_textlines_marginals_left: + ocr_textlines = ocr_all_textlines_marginals_left[mm] else: ocr_textlines = None - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter, ocr_textlines) + self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) + + for mm in range(len(found_polygons_marginals_right)): + marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_right[mm], page_coord))) + page.add_TextRegion(marginal) + if ocr_all_textlines_marginals_right: + ocr_textlines = ocr_all_textlines_marginals_right[mm] + else: + ocr_textlines = None + self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) for mm in range(len(found_polygons_drop_capitals)): dropcapital = TextRegionType(id=counter.next_region_id, type_='drop-capital', From 9b9d21d8acf4a32ae5eb888ddb33d53b701a535b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Thu, 28 Aug 2025 11:30:59 +0200 Subject: [PATCH 182/374] eynollah ocr: support using either a specific model name or a models directory (default model) --- src/eynollah/cli.py | 18 +++++++++--------- src/eynollah/eynollah.py | 28 +++++++++++++++++----------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 67fd57e..9dc326d 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -456,6 +456,11 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="directory of models", type=click.Path(exists=True, file_okay=False), ) +@click.option( + "--model_name", + help="Specific model file path to use for OCR", + type=click.Path(exists=True, file_okay=False), +) @click.option( "--tr_ocr", "-trocr/-notrocr", @@ -474,12 +479,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ is_flag=True, help="if this parameter set to true, cropped textline images will not be masked with textline contour.", ) -@click.option( - "--draw_texts_on_image", - "-dtoi/-ndtoi", - is_flag=True, - help="if this parameter set to true, the predicted texts will be displayed on an image.", -) @click.option( "--prediction_with_both_of_rgb_and_bin", "-brb/-nbrb", @@ -508,16 +507,17 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): +def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, model_name, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) + + assert not model or not model_name, "model directory -m can not be set alongside specific model name --model_name" assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text -etit can not be set alongside transformer ocr -tr_ocr" assert not export_textline_images_and_text or not model, "Exporting textline and text -etit can not be set alongside model -m" assert not export_textline_images_and_text or not batch_size, "Exporting textline and text -etit can not be set alongside batch size -bs" assert not export_textline_images_and_text or not dir_in_bin, "Exporting textline and text -etit can not be set alongside directory of bin images -dib" assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit" - assert not export_textline_images_and_text or not draw_texts_on_image, "Exporting textline and text -etit can not be set alongside draw text on image -dtoi" assert not export_textline_images_and_text or not prediction_with_both_of_rgb_and_bin, "Exporting textline and text -etit can not be set alongside prediction with both rgb and bin -brb" assert (bool(image) ^ bool(dir_in)), "Either -i (single image) or -di (directory) must be provided, but not both." eynollah_ocr = Eynollah_ocr( @@ -528,10 +528,10 @@ def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, dir_in_bin=dir_in_bin, dir_out=out, dir_models=model, + model_name=model_name, tr_ocr=tr_ocr, export_textline_images_and_text=export_textline_images_and_text, do_not_mask_with_textline_contour=do_not_mask_with_textline_contour, - draw_texts_on_image=draw_texts_on_image, prediction_with_both_of_rgb_and_bin=prediction_with_both_of_rgb_and_bin, batch_size=batch_size, pref_of_dataset=dataset_abbrevation, diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 30e180d..ec2900f 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5171,6 +5171,7 @@ class Eynollah_ocr: def __init__( self, dir_models, + model_name=None, dir_xmls=None, dir_in=None, image_filename=None, @@ -5181,7 +5182,6 @@ class Eynollah_ocr: batch_size=None, export_textline_images_and_text=False, do_not_mask_with_textline_contour=False, - draw_texts_on_image=False, prediction_with_both_of_rgb_and_bin=False, pref_of_dataset=None, min_conf_value_of_textline_text : Optional[float]=None, @@ -5193,10 +5193,10 @@ class Eynollah_ocr: self.dir_out = dir_out self.dir_xmls = dir_xmls self.dir_models = dir_models + self.model_name = model_name self.tr_ocr = tr_ocr self.export_textline_images_and_text = export_textline_images_and_text self.do_not_mask_with_textline_contour = do_not_mask_with_textline_contour - self.draw_texts_on_image = draw_texts_on_image self.dir_out_image_text = dir_out_image_text self.prediction_with_both_of_rgb_and_bin = prediction_with_both_of_rgb_and_bin self.pref_of_dataset = pref_of_dataset @@ -5210,7 +5210,10 @@ class Eynollah_ocr: if tr_ocr: self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" + if self.model_name: + self.model_ocr_dir = self.model_name + else: + self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) self.model_ocr.to(self.device) if not batch_size: @@ -5219,7 +5222,10 @@ class Eynollah_ocr: self.b_s = int(batch_size) else: - self.model_ocr_dir = dir_models + "/model_step_45000_ocr"#"/model_eynollah_ocr_cnnrnn_20250805"# + if self.model_name: + self.model_ocr_dir = self.model_name + else: + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250805" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( @@ -5230,7 +5236,7 @@ class Eynollah_ocr: else: self.b_s = int(batch_size) - with open(os.path.join(self.model_ocr_dir, "characters_20250707_all_lang.txt"),"r") as config_file: + with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: characters = json.load(config_file) AUTOTUNE = tf.data.AUTOTUNE @@ -5271,7 +5277,7 @@ class Eynollah_ocr: img = cv2.imread(dir_img) - if self.draw_texts_on_image: + if self.dir_out_image_text: out_image_with_text = os.path.join(self.dir_out_image_text, file_name+'.png') image_text = Image.new("RGB", (img.shape[1], img.shape[0]), "white") draw = ImageDraw.Draw(image_text) @@ -5306,7 +5312,7 @@ class Eynollah_ocr: textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) x,y,w,h = cv2.boundingRect(textline_coords) - if self.draw_texts_on_image: + if self.dir_out_image_text: total_bb_coordinates.append([x,y,w,h]) h2w_ratio = h/float(w) @@ -5363,7 +5369,7 @@ class Eynollah_ocr: unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - if self.draw_texts_on_image: + if self.dir_out_image_text: font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = ImageFont.truetype(font_path, 40) @@ -5463,7 +5469,7 @@ class Eynollah_ocr: dir_img_bin = os.path.join(self.dir_in_bin, file_name+'.png') img_bin = cv2.imread(dir_img_bin) - if self.draw_texts_on_image: + if self.dir_out_image_text: out_image_with_text = os.path.join(self.dir_out_image_text, file_name+'.png') image_text = Image.new("RGB", (img.shape[1], img.shape[0]), "white") draw = ImageDraw.Draw(image_text) @@ -5508,7 +5514,7 @@ class Eynollah_ocr: if type_textregion=='drop-capital': angle_degrees = 0 - if self.draw_texts_on_image: + if self.dir_out_image_text: total_bb_coordinates.append([x,y,w,h]) w_scaled = w * image_height/float(h) @@ -5829,7 +5835,7 @@ class Eynollah_ocr: unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - if self.draw_texts_on_image: + if self.dir_out_image_text: font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = ImageFont.truetype(font_path, 40) From 6a735daa606aa50e172c8cd6d82f18d94e8e9ea8 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 31 Aug 2025 23:30:54 +0200 Subject: [PATCH 183/374] Update README.md --- README.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8a2c4a4..1adc3d7 100644 --- a/README.md +++ b/README.md @@ -118,10 +118,29 @@ eynollah binarization \ ``` ### OCR -Under development +The OCR module performs text recognition from images using two main families of pretrained models: CNN-RNN–based OCR and Transformer-based OCR. + +The command-line interface for ocr can be called like this: + +```sh +eynollah ocr \ + -m | --model_name \ + -i | -di \ + -dx \ + -o +``` ### Machine-based-reading-order -Under development +The machine-based reading-order module employs a pretrained model to identify the reading order from layouts represented in PAGE-XML files. + +The command-line interface for machine based reading order can be called like this: + +```sh +eynollah machine-based-reading-order \ + -m \ + -xml | -dx \ + -o +``` #### Use as OCR-D processor Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) [processor](https://ocr-d.de/en/spec/cli), From e15640aa8aa4a3dec9f694fcca82bde9c3f516d6 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 15 Sep 2025 13:36:58 +0200 Subject: [PATCH 184/374] new page extraction model integration --- src/eynollah/eynollah.py | 200 +++++++++++++++++++++++++++++++-------- 1 file changed, 160 insertions(+), 40 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index ec2900f..3288b75 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -285,7 +285,7 @@ class Eynollah: #"/eynollah-full-regions-1column_20210425" self.model_region_dir_fully_np = dir_models + "/modelens_full_lay_1__4_3_091124" #self.model_region_dir_fully = dir_models + "/eynollah-full-regions-3+column_20210425" - self.model_page_dir = dir_models + "/eynollah-page-extraction_20210425" + self.model_page_dir = dir_models + "/model_ens_page" self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" @@ -1591,11 +1591,11 @@ class Eynollah: self.logger.debug("enter extract_page") cont_page = [] if not self.ignore_page_extraction: - img = cv2.GaussianBlur(self.image, (5, 5), 0) + img = np.copy(self.image)#cv2.GaussianBlur(self.image, (5, 5), 0) img_page_prediction = self.do_prediction(False, img, self.model_page) imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) _, thresh = cv2.threshold(imgray, 0, 255, 0) - thresh = cv2.dilate(thresh, KERNEL, iterations=3) + ##thresh = cv2.dilate(thresh, KERNEL, iterations=3) contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) if len(contours)>0: @@ -1603,24 +1603,25 @@ class Eynollah: for j in range(len(contours))]) cnt = contours[np.argmax(cnt_size)] x, y, w, h = cv2.boundingRect(cnt) - if x <= 30: - w += x - x = 0 - if (self.image.shape[1] - (x + w)) <= 30: - w = w + (self.image.shape[1] - (x + w)) - if y <= 30: - h = h + y - y = 0 - if (self.image.shape[0] - (y + h)) <= 30: - h = h + (self.image.shape[0] - (y + h)) + #if x <= 30: + #w += x + #x = 0 + #if (self.image.shape[1] - (x + w)) <= 30: + #w = w + (self.image.shape[1] - (x + w)) + #if y <= 30: + #h = h + y + #y = 0 + #if (self.image.shape[0] - (y + h)) <= 30: + #h = h + (self.image.shape[0] - (y + h)) box = [x, y, w, h] else: box = [0, 0, img.shape[1], img.shape[0]] cropped_page, page_coord = crop_image_inside_box(box, self.image) - cont_page.append(np.array([[page_coord[2], page_coord[0]], - [page_coord[3], page_coord[0]], - [page_coord[3], page_coord[1]], - [page_coord[2], page_coord[1]]])) + cont_page = cnt + #cont_page.append(np.array([[page_coord[2], page_coord[0]], + #[page_coord[3], page_coord[0]], + #[page_coord[3], page_coord[1]], + #[page_coord[2], page_coord[1]]])) self.logger.debug("exit extract_page") else: box = [0, 0, self.image.shape[1], self.image.shape[0]] @@ -3063,10 +3064,20 @@ class Eynollah: if self.plotter: self.plotter.save_page_image(image_page) - + + mask_page = np.zeros((text_regions_p_1.shape[0], text_regions_p_1.shape[1])).astype(np.int8) + mask_page = cv2.fillPoly(mask_page, pts=[cont_page], color=(1,)) + + text_regions_p_1[mask_page==0] = 0 + textline_mask_tot_ea[mask_page==0] = 0 + text_regions_p_1 = text_regions_p_1[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] textline_mask_tot_ea = textline_mask_tot_ea[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] img_bin_light = img_bin_light[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + ###text_regions_p_1 = text_regions_p_1[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + ###textline_mask_tot_ea = textline_mask_tot_ea[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + ###img_bin_light = img_bin_light[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] mask_images = (text_regions_p_1[:, :] == 2) * 1 mask_images = mask_images.astype(np.uint8) @@ -5299,8 +5310,12 @@ class Eynollah_ocr: cropped_lines = [] cropped_lines_region_indexer = [] cropped_lines_meging_indexing = [] + + extracted_texts = [] indexer_text_region = 0 + indexer_b_s = 0 + for nn in root1.iter(region_tags): for child_textregion in nn: if child_textregion.tag.endswith("TextLine"): @@ -5325,40 +5340,105 @@ class Eynollah_ocr: img_crop = img_poly_on_img[y:y+h, x:x+w, :] img_crop[mask_poly==0] = 255 + if h2w_ratio > 0.1: cropped_lines.append(resize_image(img_crop, tr_ocr_input_height_and_width, tr_ocr_input_height_and_width) ) cropped_lines_meging_indexing.append(0) + indexer_b_s+=1 + if indexer_b_s==self.b_s: + imgs = cropped_lines[:] + cropped_lines = [] + indexer_b_s = 0 + + pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values + generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + + extracted_texts = extracted_texts + generated_text_merged + else: splited_images, _ = return_textlines_split_if_needed(img_crop, None) #print(splited_images) if splited_images: cropped_lines.append(resize_image(splited_images[0], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) cropped_lines_meging_indexing.append(1) + indexer_b_s+=1 + + if indexer_b_s==self.b_s: + imgs = cropped_lines[:] + cropped_lines = [] + indexer_b_s = 0 + + pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values + generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + + extracted_texts = extracted_texts + generated_text_merged + + cropped_lines.append(resize_image(splited_images[1], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) cropped_lines_meging_indexing.append(-1) + indexer_b_s+=1 + + if indexer_b_s==self.b_s: + imgs = cropped_lines[:] + cropped_lines = [] + indexer_b_s = 0 + + pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values + generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + + extracted_texts = extracted_texts + generated_text_merged + else: cropped_lines.append(img_crop) cropped_lines_meging_indexing.append(0) + indexer_b_s+=1 + + if indexer_b_s==self.b_s: + imgs = cropped_lines[:] + cropped_lines = [] + indexer_b_s = 0 + + pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values + generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + + extracted_texts = extracted_texts + generated_text_merged + + + indexer_text_region = indexer_text_region +1 - - extracted_texts = [] - n_iterations = math.ceil(len(cropped_lines) / self.b_s) - - for i in range(n_iterations): - if i==(n_iterations-1): - n_start = i*self.b_s - imgs = cropped_lines[n_start:] - else: - n_start = i*self.b_s - n_end = (i+1)*self.b_s - imgs = cropped_lines[n_start:n_end] + if indexer_b_s!=0: + imgs = cropped_lines[:] + cropped_lines = [] + indexer_b_s = 0 + pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) extracted_texts = extracted_texts + generated_text_merged + ####extracted_texts = [] + ####n_iterations = math.ceil(len(cropped_lines) / self.b_s) + + ####for i in range(n_iterations): + ####if i==(n_iterations-1): + ####n_start = i*self.b_s + ####imgs = cropped_lines[n_start:] + ####else: + ####n_start = i*self.b_s + ####n_end = (i+1)*self.b_s + ####imgs = cropped_lines[n_start:n_end] + ####pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values + ####generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) + ####generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + + ####extracted_texts = extracted_texts + generated_text_merged + del cropped_lines gc.collect() @@ -5409,31 +5489,71 @@ class Eynollah_ocr: #print(time.time() - t0 ,'elapsed time') - indexer = 0 indexer_textregion = 0 for nn in root1.iter(region_tags): - text_subelement_textregion = ET.SubElement(nn, 'TextEquiv') - unicode_textregion = ET.SubElement(text_subelement_textregion, 'Unicode') + #id_textregion = nn.attrib['id'] + #id_textregions.append(id_textregion) + #textregions_by_existing_ids.append(text_by_textregion[indexer_textregion]) + + is_textregion_text = False + for childtest in nn: + if childtest.tag.endswith("TextEquiv"): + is_textregion_text = True + + if not is_textregion_text: + text_subelement_textregion = ET.SubElement(nn, 'TextEquiv') + unicode_textregion = ET.SubElement(text_subelement_textregion, 'Unicode') has_textline = False for child_textregion in nn: if child_textregion.tag.endswith("TextLine"): - text_subelement = ET.SubElement(child_textregion, 'TextEquiv') - unicode_textline = ET.SubElement(text_subelement, 'Unicode') - unicode_textline.text = extracted_texts_merged[indexer] + + is_textline_text = False + for childtest2 in child_textregion: + if childtest2.tag.endswith("TextEquiv"): + is_textline_text = True + + + if not is_textline_text: + text_subelement = ET.SubElement(child_textregion, 'TextEquiv') + ##text_subelement.set('conf', f"{extracted_conf_value_merged[indexer]:.2f}") + unicode_textline = ET.SubElement(text_subelement, 'Unicode') + unicode_textline.text = extracted_texts_merged[indexer] + else: + for childtest3 in child_textregion: + if childtest3.tag.endswith("TextEquiv"): + for child_uc in childtest3: + if child_uc.tag.endswith("Unicode"): + ##childtest3.set('conf', f"{extracted_conf_value_merged[indexer]:.2f}") + child_uc.text = extracted_texts_merged[indexer] + indexer = indexer + 1 has_textline = True if has_textline: - unicode_textregion.text = text_by_textregion[indexer_textregion] + if is_textregion_text: + for child4 in nn: + if child4.tag.endswith("TextEquiv"): + for childtr_uc in child4: + if childtr_uc.tag.endswith("Unicode"): + childtr_uc.text = text_by_textregion[indexer_textregion] + else: + unicode_textregion.text = text_by_textregion[indexer_textregion] indexer_textregion = indexer_textregion + 1 - - + ###sample_order = [(id_to_order[tid], text) for tid, text in zip(id_textregions, textregions_by_existing_ids) if tid in id_to_order] + + ##ordered_texts_sample = [text for _, text in sorted(sample_order)] + ##tot_page_text = ' '.join(ordered_texts_sample) + + ##for page_element in root1.iter(link+'Page'): + ##text_page = ET.SubElement(page_element, 'TextEquiv') + ##unicode_textpage = ET.SubElement(text_page, 'Unicode') + ##unicode_textpage.text = tot_page_text + ET.register_namespace("",name_space) tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) - #print("Job done in %.1fs", time.time() - t0) else: ###max_len = 280#512#280#512 ###padding_token = 1500#299#1500#299 From 0711166524fec03ccb91f564d688d127c287d75e Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 1 Sep 2025 11:37:22 +0200 Subject: [PATCH 185/374] changed the drop capitals bonding box to contour ratio threshold --- src/eynollah/utils/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index ca86047..05397d0 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -851,7 +851,8 @@ def putt_bb_of_drop_capitals_of_model_in_patches_in_layout(layout_in_patch, drop all_drop_capital_pixels = np.sum(mask_of_drop_cpaital_in_early_layout==1) percent_text_to_all_in_drop = all_drop_capital_pixels_which_is_text_in_early_lo / float(all_drop_capital_pixels) - if (areas_cnt_text[jj] * float(drop_only.shape[0] * drop_only.shape[1]) / float(w * h) > 0.6 and + + if (areas_cnt_text[jj] * float(drop_only.shape[0] * drop_only.shape[1]) / float(w * h) > 0.7 and percent_text_to_all_in_drop >= 0.3): layout_in_patch[box0] = drop_capital_label else: From 68a71be8bc77567984131dc5e16a733209bf32f2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sat, 13 Sep 2025 22:40:11 +0200 Subject: [PATCH 186/374] Running inference on files in a directory --- train/inference.py | 86 +++++++++++++++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/train/inference.py b/train/inference.py index aecd0e6..094c528 100644 --- a/train/inference.py +++ b/train/inference.py @@ -28,8 +28,9 @@ Tool to load model and predict for given image. """ class sbb_predict: - def __init__(self,image, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area): + def __init__(self,image, dir_in, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area): self.image=image + self.dir_in=dir_in self.patches=patches self.save=save self.save_layout=save_layout @@ -223,11 +224,10 @@ class sbb_predict: return added_image, layout_only - def predict(self): - self.start_new_session_and_model() + def predict(self, image_dir): if self.task == 'classification': classes_names = self.config_params_model['classification_classes_name'] - img_1ch = img=cv2.imread(self.image, 0) + img_1ch = img=cv2.imread(image_dir, 0) img_1ch = img_1ch / 255.0 img_1ch = cv2.resize(img_1ch, (self.config_params_model['input_height'], self.config_params_model['input_width']), interpolation=cv2.INTER_NEAREST) @@ -438,7 +438,7 @@ class sbb_predict: if self.patches: #def textline_contours(img,input_width,input_height,n_classes,model): - img=cv2.imread(self.image) + img=cv2.imread(image_dir) self.img_org = np.copy(img) if img.shape[0] < self.img_height: @@ -529,7 +529,7 @@ class sbb_predict: else: - img=cv2.imread(self.image) + img=cv2.imread(image_dir) self.img_org = np.copy(img) width=self.img_width @@ -557,22 +557,50 @@ class sbb_predict: def run(self): - res=self.predict() - if (self.task == 'classification' or self.task == 'reading_order'): - pass - elif self.task == 'enhancement': - if self.save: - cv2.imwrite(self.save,res) + self.start_new_session_and_model() + if self.image: + res=self.predict(image_dir = self.image) + + if (self.task == 'classification' or self.task == 'reading_order'): + pass + elif self.task == 'enhancement': + if self.save: + cv2.imwrite(self.save,res) + else: + img_seg_overlayed, only_layout = self.visualize_model_output(res, self.img_org, self.task) + if self.save: + cv2.imwrite(self.save,img_seg_overlayed) + if self.save_layout: + cv2.imwrite(self.save_layout, only_layout) + + if self.ground_truth: + gt_img=cv2.imread(self.ground_truth) + self.IoU(gt_img[:,:,0],res[:,:,0]) + else: - img_seg_overlayed, only_layout = self.visualize_model_output(res, self.img_org, self.task) - if self.save: - cv2.imwrite(self.save,img_seg_overlayed) - if self.save_layout: - cv2.imwrite(self.save_layout, only_layout) + ls_images = os.listdir(self.dir_in) + for ind_image in ls_images: + f_name = ind_image.split('.')[0] + image_dir = os.path.join(self.dir_in, ind_image) + res=self.predict(image_dir) - if self.ground_truth: - gt_img=cv2.imread(self.ground_truth) - self.IoU(gt_img[:,:,0],res[:,:,0]) + if (self.task == 'classification' or self.task == 'reading_order'): + pass + elif self.task == 'enhancement': + self.save = os.path.join(self.out, f_name+'.png') + cv2.imwrite(self.save,res) + else: + img_seg_overlayed, only_layout = self.visualize_model_output(res, self.img_org, self.task) + self.save = os.path.join(self.out, f_name+'_overlayed.png') + cv2.imwrite(self.save,img_seg_overlayed) + self.save_layout = os.path.join(self.out, f_name+'_layout.png') + cv2.imwrite(self.save_layout, only_layout) + + if self.ground_truth: + gt_img=cv2.imread(self.ground_truth) + self.IoU(gt_img[:,:,0],res[:,:,0]) + + @click.command() @click.option( @@ -581,6 +609,12 @@ class sbb_predict: help="image filename", type=click.Path(exists=True, dir_okay=False), ) +@click.option( + "--dir_in", + "-di", + help="directory of images", + type=click.Path(exists=True, file_okay=False), +) @click.option( "--out", "-o", @@ -626,15 +660,19 @@ class sbb_predict: "-min", help="min area size of regions considered for reading order detection. The default value is zero and means that all text regions are considered for reading order.", ) -def main(image, model, patches, save, save_layout, ground_truth, xml_file, out, min_area): +def main(image, dir_in, model, patches, save, save_layout, ground_truth, xml_file, out, min_area): + assert image or dir_in, "Either a single image -i or a dir_in -di is required" with open(os.path.join(model,'config.json')) as f: config_params_model = json.load(f) task = config_params_model['task'] if (task != 'classification' and task != 'reading_order'): - if not save: - print("Error: You used one of segmentation or binarization task but not set -s, you need a filename to save visualized output with -s") + if image and not save: + print("Error: You used one of segmentation or binarization task with image input but not set -s, you need a filename to save visualized output with -s") sys.exit(1) - x=sbb_predict(image, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area) + if dir_in and not out: + print("Error: You used one of segmentation or binarization task with dir_in but not set -out") + sys.exit(1) + x=sbb_predict(image, dir_in, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area) x.run() if __name__=="__main__": From 542646791ded9d40ef238dfae595c40ac2a6adcc Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 3 Sep 2025 19:18:11 +0200 Subject: [PATCH 187/374] For TrOCR, the cropped text lines will no longer be added to a list before prediction. Instead, for each batch size, the text line images will be collected and predictions will be made directly on them. --- src/eynollah/utils/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 05397d0..ca86047 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -851,8 +851,7 @@ def putt_bb_of_drop_capitals_of_model_in_patches_in_layout(layout_in_patch, drop all_drop_capital_pixels = np.sum(mask_of_drop_cpaital_in_early_layout==1) percent_text_to_all_in_drop = all_drop_capital_pixels_which_is_text_in_early_lo / float(all_drop_capital_pixels) - - if (areas_cnt_text[jj] * float(drop_only.shape[0] * drop_only.shape[1]) / float(w * h) > 0.7 and + if (areas_cnt_text[jj] * float(drop_only.shape[0] * drop_only.shape[1]) / float(w * h) > 0.6 and percent_text_to_all_in_drop >= 0.3): layout_in_patch[box0] = drop_capital_label else: From 310679eeb8c97562cbcd0da6462356ec1d58aa8f Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Tue, 16 Sep 2025 14:27:15 +0200 Subject: [PATCH 188/374] page extraction model name is changed --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 3288b75..7ef2361 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -285,7 +285,7 @@ class Eynollah: #"/eynollah-full-regions-1column_20210425" self.model_region_dir_fully_np = dir_models + "/modelens_full_lay_1__4_3_091124" #self.model_region_dir_fully = dir_models + "/eynollah-full-regions-3+column_20210425" - self.model_page_dir = dir_models + "/model_ens_page" + self.model_page_dir = dir_models + "/model_eynollah_page_extraction_20250915" self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" From c64d1026136161ee9c2ea71e9cb996390531c9de Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 18 Sep 2025 13:07:41 +0200 Subject: [PATCH 189/374] move logging to CLI and make initialization optional --- .gitignore | 1 + src/eynollah/cli.py | 54 +++++++++++++++++++++++++++++++++++++--- src/eynollah/eynollah.py | 13 +--------- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 5236dde..0d5d834 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ models_eynollah* output.html /build /dist +*.tif diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index c189aca..b980e16 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -1,5 +1,6 @@ import sys import click +import logging from ocrd_utils import initLogging, getLevelName, getLogger from eynollah.eynollah import Eynollah, Eynollah_ocr from eynollah.sbb_binarize import SbbBinarizer @@ -241,15 +242,61 @@ def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out) is_flag=True, help="if this parameter set to true, this tool will ignore layout detection and reading order. It means that textline detection will be done within printspace and contours of textline will be written in xml output file.", ) +# TODO move to top-level CLI context @click.option( "--log_level", "-l", type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", + help="Override 'eynollah' log level globally to this", +) +# +@click.option( + "--setup-logging", + is_flag=True, + help="Setup a basic console logger", ) -def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, num_col_upper, num_col_lower, skip_layout_and_reading_order, ignore_page_extraction, log_level): - initLogging() +def layout( + image, + out, + overwrite, + dir_in, + model, + save_images, + save_layout, + save_deskewed, + save_all, + extract_only_images, + save_page, + enable_plotting, + allow_enhancement, + curved_line, + textline_light, + full_layout, + tables, + right2left, + input_binary, + allow_scaling, + headers_off, + light_version, + reading_order_machine_based, + do_ocr, + num_col_upper, + num_col_lower, + skip_layout_and_reading_order, + ignore_page_extraction, + log_level, + setup_logging +): + if setup_logging: + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(message)s') + console_handler.setFormatter(formatter) + getLogger('eynollah').addHandler(console_handler) + getLogger('eynollah').setLevel(logging.INFO) + else: + initLogging() if log_level: getLogger('eynollah').setLevel(getLevelName(log_level)) assert enable_plotting or not save_layout, "Plotting with -sl also requires -ep" @@ -273,7 +320,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ assert image or dir_in, "Either a single image -i or a dir_in -di is required" eynollah = Eynollah( model, - logger=getLogger('eynollah'), dir_out=out, dir_of_cropped_images=save_images, extract_only_images=extract_only_images, diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index d9939ca..e80b8d0 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -6,7 +6,6 @@ document layout analysis (segmentation) with output in PAGE-XML """ -from logging import Logger from difflib import SequenceMatcher as sq from PIL import Image, ImageDraw, ImageFont import math @@ -201,18 +200,8 @@ class Eynollah: num_col_upper : Optional[int] = None, num_col_lower : Optional[int] = None, skip_layout_and_reading_order : bool = False, - logger : Optional[Logger] = None, ): - if logger: - self.logger = logger - else: - self.logger = getLogger('eynollah') - console_handler = logging.StreamHandler(sys.stdout) - console_handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(message)s') - console_handler.setFormatter(formatter) - self.logger.addHandler(console_handler) - self.logger.setLevel(logging.INFO) + self.logger = getLogger('eynollah') if skip_layout_and_reading_order: textline_light = True From 146102842aac15275647e2e565e5e2549b3ba1fd Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 18 Sep 2025 13:15:18 +0200 Subject: [PATCH 190/374] convert all print stmts to logger.info calls --- src/eynollah/eynollah.py | 223 ++++++++++----------------------------- 1 file changed, 56 insertions(+), 167 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index e80b8d0..39476e2 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -312,9 +312,7 @@ class Eynollah: except: self.logger.warning("no GPU device available") - msg = "Loading models..." - print(msg) - self.logger.info(msg) + self.logger.info("Loading models...") self.model_page = self.our_load_model(self.model_page_dir) self.model_classifier = self.our_load_model(self.model_dir_of_col_classifier) @@ -343,9 +341,7 @@ class Eynollah: if self.tables: self.model_table = self.our_load_model(self.model_table_dir) - msg = f"Model initialization complete ({time.time() - t_start:.1f}s)" - print(msg) - self.logger.info(msg) + self.logger.info(f"Model initialization complete ({time.time() - t_start:.1f}s)") def cache_images(self, image_filename=None, image_pil=None, dpi=None): ret = {} @@ -3453,7 +3449,7 @@ class Eynollah: peaks_real, _ = find_peaks(sum_smoothed, height=0) if len(peaks_real)>70: - print(len(peaks_real), 'len(peaks_real)') + self.logger.debug(f'len(peaks_real) = {len(peaks_real)}') peaks_real = peaks_real[(peaks_realwidth1)] @@ -4302,14 +4298,11 @@ class Eynollah: if dir_in: self.logger.info("All jobs done in %.1fs", time.time() - t0_tot) - print("all Job done in %.1fs", time.time() - t0_tot) def run_single(self): t0 = time.time() - msg = f"Processing file: {self.writer.image_filename}" - print(msg) - self.logger.info(msg) + self.logger.info(f"Processing file: {self.writer.image_filename}") # Log enabled features directly enabled_modes = [] @@ -4325,35 +4318,23 @@ class Eynollah: enabled_modes.append("Table detection") if enabled_modes: - msg = "Enabled modes: " + ", ".join(enabled_modes) - print(msg) - self.logger.info(msg) + self.logger.info("Enabled modes: " + ", ".join(enabled_modes)) - msg = "Step 1/5: Image Enhancement" - print(msg) - self.logger.info(msg) + self.logger.info("Step 1/5: Image Enhancement") img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) - msg = f"Image: {self.image.shape[1]}x{self.image.shape[0]}, {self.dpi} DPI, {num_col_classifier} columns" - print(msg) - self.logger.info(msg) + self.logger.info(f"Image: {self.image.shape[1]}x{self.image.shape[0]}, {self.dpi} DPI, {num_col_classifier} columns") if is_image_enhanced: - msg = "Enhancement applied" - print(msg) - self.logger.info(msg) + self.logger.info("Enhancement applied") - msg = f"Enhancement complete ({time.time() - t0:.1f}s)" - print(msg) - self.logger.info(msg) + self.logger.info(f"Enhancement complete ({time.time() - t0:.1f}s)") # Image Extraction Mode if self.extract_only_images: - msg = "Step 2/5: Image Extraction Mode" - print(msg) - self.logger.info(msg) + self.logger.info("Step 2/5: Image Extraction Mode") text_regions_p_1, erosion_hurts, polygons_lines_xml, polygons_of_images, image_page, page_coord, cont_page = \ self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) @@ -4367,19 +4348,13 @@ class Eynollah: if self.plotter: self.plotter.write_images_into_directory(polygons_of_images, image_page) - msg = "Image extraction complete" - print(msg) - self.logger.info(msg) + self.logger.info("Image extraction complete") return pcgts # Basic Processing Mode if self.skip_layout_and_reading_order: - msg = "Step 2/5: Basic Processing Mode" - print(msg) - self.logger.info(msg) - msg = "Skipping layout analysis and reading order detection" - print(msg) - self.logger.info(msg) + self.logger.info("Step 2/5: Basic Processing Mode") + self.logger.info("Skipping layout analysis and reading order detection") _ ,_, _, textline_mask_tot_ea, img_bin_light, _ = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier, @@ -4421,21 +4396,15 @@ class Eynollah: all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) - msg = "Basic processing complete" - print(msg) - self.logger.info(msg) + self.logger.info("Basic processing complete") return pcgts #print("text region early -1 in %.1fs", time.time() - t0) t1 = time.time() - msg = "Step 2/5: Layout Analysis" - print(msg) - self.logger.info(msg) + self.logger.info("Step 2/5: Layout Analysis") if self.light_version: - msg = "Using light version processing" - print(msg) - self.logger.info(msg) + self.logger.info("Using light version processing") text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier) #print("text region early -2 in %.1fs", time.time() - t0) @@ -4466,29 +4435,21 @@ class Eynollah: text_regions_p_1 ,erosion_hurts, polygons_lines_xml = \ self.get_regions_from_xy_2models(img_res, is_image_enhanced, num_col_classifier) - msg = f"Textregion detection took {time.time() - t1:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Textregion detection took {time.time() - t1:.1f}s") confidence_matrix = np.zeros((text_regions_p_1.shape[:2])) t1 = time.time() num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ text_regions_p_1, cont_page, table_prediction = \ self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts) - msg = f"Graphics detection took {time.time() - t1:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Graphics detection took {time.time() - t1:.1f}s") #self.logger.info('cont_page %s', cont_page) #plt.imshow(table_prediction) #plt.show() - msg = f"Layout analysis complete ({time.time() - t1:.1f}s)" - print(msg) - self.logger.info(msg) + self.logger.info(f"Layout analysis complete ({time.time() - t1:.1f}s)") if not num_col: - msg = "No columns detected - generating empty PAGE-XML" - print(msg) - self.logger.info(msg) + self.logger.info("No columns detected - generating empty PAGE-XML") ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( @@ -4500,18 +4461,12 @@ class Eynollah: t1 = time.time() if not self.light_version: textline_mask_tot_ea = self.run_textline(image_page) - msg = f"Textline detection took {time.time() - t1:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Textline detection took {time.time() - t1:.1f}s") t1 = time.time() slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) if np.abs(slope_deskew) > 0.01: # Only log if there is significant skew - msg = f"Applied deskew correction: {slope_deskew:.2f} degrees" - print(msg) - self.logger.info(msg) - msg = f"Deskewing took {time.time() - t1:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Applied deskew correction: {slope_deskew:.2f} degrees") + self.logger.info(f"Deskewing took {time.time() - t1:.1f}s") elif num_col_classifier in (1,2): org_h_l_m = textline_mask_tot_ea.shape[0] org_w_l_m = textline_mask_tot_ea.shape[1] @@ -4532,18 +4487,12 @@ class Eynollah: self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction) - msg = "Step 3/5: Text Line Detection" - print(msg) - self.logger.info(msg) + self.logger.info("Step 3/5: Text Line Detection") if self.curved_line: - msg = "Mode: Curved line detection" - print(msg) - self.logger.info(msg) + self.logger.info("Mode: Curved line detection") elif self.textline_light: - msg = "Mode: Light detection" - print(msg) - self.logger.info(msg) + self.logger.info("Mode: Light detection") if self.light_version and num_col_classifier in (1,2): image_page = resize_image(image_page,org_h_l_m, org_w_l_m ) @@ -4554,9 +4503,7 @@ class Eynollah: table_prediction = resize_image(table_prediction,org_h_l_m, org_w_l_m ) image_page_rotated = resize_image(image_page_rotated,org_h_l_m, org_w_l_m ) - msg = f"Detection of marginals took {time.time() - t1:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Detection of marginals took {time.time() - t1:.1f}s") #print("text region early 2 marginal in %.1fs", time.time() - t0) ## birdan sora chock chakir t1 = time.time() @@ -4655,9 +4602,7 @@ class Eynollah: cx_bigest_d_big[0] = cx_bigest_d[ind_largest] cy_biggest_d_big[0] = cy_biggest_d[ind_largest] except Exception as why: - msg = str(why) - print(f"Error: {msg}") - self.logger.error(msg) + self.logger.error(str(why)) (h, w) = text_only.shape[:2] center = (w // 2.0, h // 2.0) @@ -4875,22 +4820,14 @@ class Eynollah: t_order = time.time() if self.full_layout: - msg = "Step 4/5: Reading Order Detection" - print(msg) - self.logger.info(msg) + self.logger.info(ep 4/5: Reading Order Detection") if self.reading_order_machine_based: - msg = "Using machine-based detection" - print(msg) - self.logger.info(msg) + self.logger.info("Using machine-based detection") if self.right2left: - msg = "Right-to-left mode enabled" - print(msg) - self.logger.info(msg) + self.logger.info("Right-to-left mode enabled") if self.headers_off: - msg = "Headers ignored in reading order" - print(msg) - self.logger.info(msg) + self.logger.info("Headers ignored in reading order") if self.reading_order_machine_based: order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( @@ -4902,31 +4839,21 @@ class Eynollah: else: order_text_new, id_of_texts_tot = self.do_order_of_regions( contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) - msg = f"Detection of reading order took {time.time() - t_order:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") if self.ocr: - msg = "Step 4.5/5: OCR Processing" - print(msg) - self.logger.info(msg) + self.logger.info("Step 4.5/5: OCR Processing") if torch.cuda.is_available(): - msg = "Using GPU acceleration" - print(msg) - self.logger.info(msg) + self.logger.info("Using GPU acceleration") else: - msg = "Using CPU processing" - print(msg) - self.logger.info(msg) + self.logger.info("Using CPU processing") ocr_all_textlines = [] else: ocr_all_textlines = None - msg = "Step 5/5: Output Generation" - print(msg) - self.logger.info(msg) + self.logger.info("Step 5/5: Output Generation") output_config = [] if self.enable_plotting: @@ -4963,22 +4890,14 @@ class Eynollah: return pcgts contours_only_text_parent_h = None - msg = "Step 4/5: Reading Order Detection" - print(msg) - self.logger.info(msg) + self.logger.info("Step 4/5: Reading Order Detection") if self.reading_order_machine_based: - msg = "Using machine-based detection" - print(msg) - self.logger.info(msg) + self.logger.info("Using machine-based detection") if self.right2left: - msg = "Right-to-left mode enabled" - print(msg) - self.logger.info(msg) + self.logger.info("Right-to-left mode enabled") if self.headers_off: - msg = "Headers ignored in reading order" - print(msg) - self.logger.info(msg) + self.logger.info("Headers ignored in reading order") if self.reading_order_machine_based: order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( @@ -5000,32 +4919,20 @@ class Eynollah: contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) if self.ocr: - msg = "Step 4.5/5: OCR Processing" - print(msg) - self.logger.info(msg) + self.logger.info("Step 4.5/5: OCR Processing") if torch.cuda.is_available(): - msg = "Using GPU acceleration" - print(msg) - self.logger.info(msg) + self.logger.info("Using GPU acceleration") else: - msg = "Using CPU processing" - print(msg) - self.logger.info(msg) + self.logger.info("Using CPU processing") if self.light_version: - msg = "Using light version OCR" - print(msg) - self.logger.info(msg) + self.logger.info("Using light version OCR") if self.textline_light: - msg = "Using light text line detection for OCR" - print(msg) - self.logger.info(msg) + self.logger.info("Using light text line detection for OCR") - msg = "Processing text lines..." - print(msg) - self.logger.info(msg) + self.logger.info("Processing text lines...") device = cuda.get_current_device() device.reset() @@ -5077,37 +4984,23 @@ class Eynollah: else: ocr_all_textlines = None #print(ocr_all_textlines) - msg = f"Detection of reading order took {time.time() - t_order:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") - msg = "Step 5/5: Output Generation" - print(msg) - self.logger.info(msg) + self.logger.info("Step 5/5: Output Generation") - msg = "Generating PAGE-XML output" - print(msg) - self.logger.info(msg) + self.logger.info("Generating PAGE-XML output") if self.enable_plotting: - msg = "Saving debug plots" - print(msg) - self.logger.info(msg) + self.logger.info("Saving debug plots") if self.dir_of_cropped_images: - msg = f"Saving cropped images to: {self.dir_of_cropped_images}" - print(msg) - self.logger.info(msg) + self.logger.info(f"Saving cropped images to: {self.dir_of_cropped_images}") if self.dir_of_layout: - msg = f"Saving layout plots to: {self.dir_of_layout}" - print(msg) - self.logger.info(msg) + self.logger.info(f"Saving layout plots to: {self.dir_of_layout}") if self.dir_of_deskewed: - msg = f"Saving deskewed images to: {self.dir_of_deskewed}" - print(msg) - self.logger.info(msg) + self.logger.info(f"Saving deskewed images to: {self.dir_of_deskewed}") pcgts = self.writer.build_pagexml_no_full_layout( txt_con_org, page_coord, order_text_new, id_of_texts_tot, @@ -5115,13 +5008,9 @@ class Eynollah: all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) - msg = f"\nProcessing completed in {time.time() - t0:.1f}s" - print(msg) - self.logger.info(msg) + self.logger.info(f"\nProcessing completed in {time.time() - t0:.1f}s") - msg = f"Output file: {self.writer.output_filename}" - print(msg) - self.logger.info(msg) + self.logger.info(f"Output file: {self.writer.output_filename}") return pcgts From 5c9cf8472bc3c39827d751db8d1562afe02b13c3 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 18 Sep 2025 13:19:57 +0200 Subject: [PATCH 191/374] remove redundant/brittle interval logging --- src/eynollah/eynollah.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 39476e2..14dfbb3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4293,7 +4293,6 @@ class Eynollah: pcgts = self.run_single() self.logger.info("Job done in %.1fs", time.time() - t0) - #print("Job done in %.1fs" % (time.time() - t0)) self.writer.write_pagexml(pcgts) if dir_in: @@ -4504,7 +4503,6 @@ class Eynollah: image_page_rotated = resize_image(image_page_rotated,org_h_l_m, org_w_l_m ) self.logger.info(f"Detection of marginals took {time.time() - t1:.1f}s") - #print("text region early 2 marginal in %.1fs", time.time() - t0) ## birdan sora chock chakir t1 = time.time() if not self.full_layout: @@ -5008,8 +5006,6 @@ class Eynollah: all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) - self.logger.info(f"\nProcessing completed in {time.time() - t0:.1f}s") - self.logger.info(f"Output file: {self.writer.output_filename}") return pcgts From 530897c6c2a9455d3c7713257f15351de8732b99 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 19 Sep 2025 13:20:26 +0200 Subject: [PATCH 192/374] renaming argument names --- train/generate_gt_for_training.py | 34 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 91ee2c8..7810cd7 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -157,6 +157,7 @@ def image_enhancement(dir_imgs, dir_out_images, dir_out_labels, scales): def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width, min_area_size, min_area_early): xml_files_ind = os.listdir(dir_xml) + xml_files_ind = [ind_xml for ind_xml in xml_files_ind if ind_xml.endswith('.xml')] input_height = int(input_height) input_width = int(input_width) min_area = float(min_area_size) @@ -268,14 +269,14 @@ def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, i @click.option( "--dir_out", - "-do", + "-o", help="directory where plots will be written", type=click.Path(exists=True, file_okay=False), ) @click.option( "--dir_imgs", - "-dimg", + "-di", help="directory where the overlayed plots will be written", ) def visualize_reading_order(xml_file, dir_xml, dir_out, dir_imgs): @@ -283,6 +284,7 @@ def visualize_reading_order(xml_file, dir_xml, dir_out, dir_imgs): if dir_xml: xml_files_ind = os.listdir(dir_xml) + xml_files_ind = [ind_xml for ind_xml in xml_files_ind if ind_xml.endswith('.xml')] else: xml_files_ind = [xml_file] @@ -353,6 +355,12 @@ def visualize_reading_order(xml_file, dir_xml, dir_out, dir_imgs): @main.command() +@click.option( + "--xml_file", + "-xml", + help="xml filename", + type=click.Path(exists=True, dir_okay=False), +) @click.option( "--dir_xml", "-dx", @@ -362,18 +370,24 @@ def visualize_reading_order(xml_file, dir_xml, dir_out, dir_imgs): @click.option( "--dir_out", - "-do", + "-o", help="directory where plots will be written", type=click.Path(exists=True, file_okay=False), ) @click.option( "--dir_imgs", - "-dimg", + "-di", help="directory of images where textline segmentation will be overlayed", ) -def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): - xml_files_ind = os.listdir(dir_xml) +def visualize_textline_segmentation(xml_file, dir_xml, dir_out, dir_imgs): + assert xml_file or dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" + if dir_xml: + xml_files_ind = os.listdir(dir_xml) + xml_files_ind = [ind_xml for ind_xml in xml_files_ind if ind_xml.endswith('.xml')] + else: + xml_files_ind = [xml_file] + for ind_xml in tqdm(xml_files_ind): indexer = 0 #print(ind_xml) @@ -408,20 +422,21 @@ def visualize_textline_segmentation(dir_xml, dir_out, dir_imgs): @click.option( "--dir_out", - "-do", + "-o", help="directory where plots will be written", type=click.Path(exists=True, file_okay=False), ) @click.option( "--dir_imgs", - "-dimg", + "-di", help="directory of images where textline segmentation will be overlayed", ) def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): assert xml_file or dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" if dir_xml: xml_files_ind = os.listdir(dir_xml) + xml_files_ind = [ind_xml for ind_xml in xml_files_ind if ind_xml.endswith('.xml')] else: xml_files_ind = [xml_file] @@ -466,7 +481,7 @@ def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): @click.option( "--dir_out", - "-do", + "-o", help="directory where plots will be written", type=click.Path(exists=True, file_okay=False), ) @@ -476,6 +491,7 @@ def visualize_ocr_text(xml_file, dir_xml, dir_out): assert xml_file or dir_xml, "A single xml file -xml or a dir of xml files -dx is required not both of them" if dir_xml: xml_files_ind = os.listdir(dir_xml) + xml_files_ind = [ind_xml for ind_xml in xml_files_ind if ind_xml.endswith('.xml')] else: xml_files_ind = [xml_file] From 994bc8a1c07270cd390a59860a18e878fed1da1d Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 19 Sep 2025 15:24:34 +0200 Subject: [PATCH 193/374] debug new page extraction in the case of ignoring page extraction --- src/eynollah/eynollah.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 7ef2361..07cf8d9 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3065,11 +3065,12 @@ class Eynollah: if self.plotter: self.plotter.save_page_image(image_page) - mask_page = np.zeros((text_regions_p_1.shape[0], text_regions_p_1.shape[1])).astype(np.int8) - mask_page = cv2.fillPoly(mask_page, pts=[cont_page], color=(1,)) - - text_regions_p_1[mask_page==0] = 0 - textline_mask_tot_ea[mask_page==0] = 0 + if not self.ignore_page_extraction: + mask_page = np.zeros((text_regions_p_1.shape[0], text_regions_p_1.shape[1])).astype(np.int8) + mask_page = cv2.fillPoly(mask_page, pts=[cont_page], color=(1,)) + + text_regions_p_1[mask_page==0] = 0 + textline_mask_tot_ea[mask_page==0] = 0 text_regions_p_1 = text_regions_p_1[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] textline_mask_tot_ea = textline_mask_tot_ea[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] From b38331b4aba9aa3db769e4f53ba9423beeb790ab Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 19 Sep 2025 18:06:18 +0200 Subject: [PATCH 194/374] writing page contour correctly in xml output + ignore unsupported file types when loading images --- src/eynollah/eynollah.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 07cf8d9..bd8f088 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1617,7 +1617,7 @@ class Eynollah: else: box = [0, 0, img.shape[1], img.shape[0]] cropped_page, page_coord = crop_image_inside_box(box, self.image) - cont_page = cnt + cont_page = [cnt] #cont_page.append(np.array([[page_coord[2], page_coord[0]], #[page_coord[3], page_coord[0]], #[page_coord[3], page_coord[1]], @@ -3067,7 +3067,7 @@ class Eynollah: if not self.ignore_page_extraction: mask_page = np.zeros((text_regions_p_1.shape[0], text_regions_p_1.shape[1])).astype(np.int8) - mask_page = cv2.fillPoly(mask_page, pts=[cont_page], color=(1,)) + mask_page = cv2.fillPoly(mask_page, pts=[cont_page[0]], color=(1,)) text_regions_p_1[mask_page==0] = 0 textline_mask_tot_ea[mask_page==0] = 0 @@ -4526,6 +4526,7 @@ class Eynollah: if dir_in: self.ls_imgs = os.listdir(dir_in) + self.ls_imgs = [ind_img for ind_img in self.ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG')] elif image_filename: self.ls_imgs = [image_filename] else: @@ -5265,6 +5266,7 @@ class Eynollah_ocr: def run(self, overwrite : bool = False): if self.dir_in: ls_imgs = os.listdir(self.dir_in) + ls_imgs = [ind_img for ind_img in ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG')] else: ls_imgs = [self.image_filename] From e97e3ab192695d0b85395990709fb70d76a0881b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Fri, 19 Sep 2025 23:23:30 +0200 Subject: [PATCH 195/374] Merge text of textlines and handle hyphenated words by joining them correctly --- src/eynollah/eynollah.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index bd8f088..1781c04 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5481,17 +5481,31 @@ class Eynollah_ocr: image_text.save(out_image_with_text) #print(len(unique_cropped_lines_region_indexer), 'unique_cropped_lines_region_indexer') + #######text_by_textregion = [] + #######for ind in unique_cropped_lines_region_indexer: + #######extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] + + #######text_by_textregion.append(" ".join(extracted_texts_merged_un)) + text_by_textregion = [] for ind in unique_cropped_lines_region_indexer: extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] - - text_by_textregion.append(" ".join(extracted_texts_merged_un)) - - #print(len(text_by_textregion) , indexer_text_region, "text_by_textregion") - - - #print(time.time() - t0 ,'elapsed time') - + if len(extracted_texts_merged_un)>1: + text_by_textregion_ind = "" + next_glue = "" + for indt in range(len(extracted_texts_merged_un)): + if extracted_texts_merged_un[indt].endswith('⸗') or extracted_texts_merged_un[indt].endswith('-') or extracted_texts_merged_un[indt].endswith('¬'): + text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt][:-1] + next_glue = "" + else: + text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt] + next_glue = " " + text_by_textregion.append(text_by_textregion_ind) + + else: + text_by_textregion.append(" ".join(extracted_texts_merged_un)) + + indexer = 0 indexer_textregion = 0 for nn in root1.iter(region_tags): @@ -5993,7 +6007,7 @@ class Eynollah_ocr: text_by_textregion_ind = "" next_glue = "" for indt in range(len(extracted_texts_merged_un)): - if extracted_texts_merged_un[indt].endswith('⸗') or extracted_texts_merged_un[indt].endswith('-'): + if extracted_texts_merged_un[indt].endswith('⸗') or extracted_texts_merged_un[indt].endswith('-') or extracted_texts_merged_un[indt].endswith('¬'): text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt][:-1] next_glue = "" else: From 6bbdfe10744dc1e9aaddd993bc8565e1b7739f7b Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 21 Sep 2025 02:32:40 +0200 Subject: [PATCH 196/374] extending image types --- src/eynollah/eynollah.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 1781c04..64e57a3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4526,7 +4526,7 @@ class Eynollah: if dir_in: self.ls_imgs = os.listdir(dir_in) - self.ls_imgs = [ind_img for ind_img in self.ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG')] + self.ls_imgs = [ind_img for ind_img in self.ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG') or ind_img.endswith('.TIF') or ind_img.endswith('.TIFF') or ind_img.endswith('.PNG')] elif image_filename: self.ls_imgs = [image_filename] else: @@ -5266,7 +5266,7 @@ class Eynollah_ocr: def run(self, overwrite : bool = False): if self.dir_in: ls_imgs = os.listdir(self.dir_in) - ls_imgs = [ind_img for ind_img in ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG')] + ls_imgs = [ind_img for ind_img in ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG') or ind_img.endswith('.TIF') or ind_img.endswith('.TIFF') or ind_img.endswith('.PNG')] else: ls_imgs = [self.image_filename] From 554f3988c9d3c9d092712dd0998e8287b951cdeb Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Sun, 21 Sep 2025 16:33:14 +0200 Subject: [PATCH 197/374] default cnn-rnn and transformer ocr models have changed to model_eynollah_ocr_cnnrnn_20250904 and model_eynollah_ocr_trocr_20250919 respectively --- src/eynollah/eynollah.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 64e57a3..574d823 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -317,9 +317,9 @@ class Eynollah: #"/eynollah-textline_20210425" self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024" if self.ocr and self.tr: - self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_trocr_20250919" elif self.ocr and not self.tr: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250805" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250904" if self.tables: if self.light_version: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" @@ -5226,7 +5226,7 @@ class Eynollah_ocr: if self.model_name: self.model_ocr_dir = self.model_name else: - self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_trocr_20250919" self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) self.model_ocr.to(self.device) if not batch_size: @@ -5238,7 +5238,7 @@ class Eynollah_ocr: if self.model_name: self.model_ocr_dir = self.model_name else: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250805" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250904" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( From a65405bead03f386cf3935df4dd58b1985cfcd21 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 22 Sep 2025 15:56:14 +0200 Subject: [PATCH 198/374] tables are visulaized within layout --- train/generate_gt_for_training.py | 2 +- train/gt_gen_utils.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/train/generate_gt_for_training.py b/train/generate_gt_for_training.py index 7810cd7..388fced 100644 --- a/train/generate_gt_for_training.py +++ b/train/generate_gt_for_training.py @@ -458,7 +458,7 @@ def visualize_layout_segmentation(xml_file, dir_xml, dir_out, dir_imgs): co_text, co_graphic, co_sep, co_img, co_table, co_noise, y_len, x_len = get_layout_contours_for_visualization(xml_file) - added_image = visualize_image_from_contours_layout(co_text['paragraph'], co_text['header']+co_text['heading'], co_text['drop-capital'], co_sep, co_img, co_text['marginalia'], img) + added_image = visualize_image_from_contours_layout(co_text['paragraph'], co_text['header']+co_text['heading'], co_text['drop-capital'], co_sep, co_img, co_text['marginalia'], co_table, img) cv2.imwrite(os.path.join(dir_out, f_name+'.png'), added_image) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 753b0f5..38d48ca 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -18,7 +18,7 @@ with warnings.catch_warnings(): warnings.simplefilter("ignore") -def visualize_image_from_contours_layout(co_par, co_header, co_drop, co_sep, co_image, co_marginal, img): +def visualize_image_from_contours_layout(co_par, co_header, co_drop, co_sep, co_image, co_marginal, co_table, img): alpha = 0.5 blank_image = np.ones( (img.shape[:]), dtype=np.uint8) * 255 @@ -30,6 +30,7 @@ def visualize_image_from_contours_layout(co_par, co_header, co_drop, co_sep, co_ col_image = (0, 100, 0) col_sep = (255, 0, 0) col_marginal = (106, 90, 205) + col_table = (0, 90, 205) if len(co_image)>0: cv2.drawContours(blank_image, co_image, -1, col_image, thickness=cv2.FILLED) # Fill the contour @@ -51,6 +52,9 @@ def visualize_image_from_contours_layout(co_par, co_header, co_drop, co_sep, co_ if len(co_marginal)>0: cv2.drawContours(blank_image, co_marginal, -1, col_marginal, thickness=cv2.FILLED) # Fill the contour + + if len(co_table)>0: + cv2.drawContours(blank_image, co_table, -1, col_table, thickness=cv2.FILLED) # Fill the contour img_final =cv2.cvtColor(blank_image, cv2.COLOR_BGR2RGB) From d0817f5744f4e78f3880d1ea87423e8260da9a81 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 12:08:50 +0200 Subject: [PATCH 199/374] fix typo --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 73d07b5..2813c56 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5091,7 +5091,7 @@ class Eynollah: t_order = time.time() if self.full_layout: - self.logger.info(ep 4/5: Reading Order Detection") + self.logger.info("Step 4/5: Reading Order Detection") if self.reading_order_machine_based: self.logger.info("Using machine-based detection") From 7933b103f5378f025eda2f5347095ee26e3eb159 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 12:09:30 +0200 Subject: [PATCH 200/374] log modes only once (in run, not in run_single) --- src/eynollah/eynollah.py | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 2813c56..82073c3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4531,6 +4531,21 @@ class Eynollah: self.logger.debug("enter run") t0_tot = time.time() + # Log enabled features directly + enabled_modes = [] + if self.light_version: + enabled_modes.append("Light version") + if self.textline_light: + enabled_modes.append("Light textline detection") + if self.full_layout: + enabled_modes.append("Full layout analysis") + if self.ocr: + enabled_modes.append("OCR") + if self.tables: + enabled_modes.append("Table detection") + if enabled_modes: + self.logger.info("Enabled modes: " + ", ".join(enabled_modes)) + if dir_in: self.ls_imgs = os.listdir(dir_in) self.ls_imgs = [ind_img for ind_img in self.ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG') or ind_img.endswith('.TIF') or ind_img.endswith('.TIFF') or ind_img.endswith('.PNG')] @@ -4563,25 +4578,7 @@ class Eynollah: def run_single(self): t0 = time.time() - self.logger.info(f"Processing file: {self.writer.image_filename}") - - # Log enabled features directly - enabled_modes = [] - if self.light_version: - enabled_modes.append("Light version") - if self.textline_light: - enabled_modes.append("Light textline detection") - if self.full_layout: - enabled_modes.append("Full layout analysis") - if self.ocr: - enabled_modes.append("OCR") - if self.tables: - enabled_modes.append("Table detection") - - if enabled_modes: - self.logger.info("Enabled modes: " + ", ".join(enabled_modes)) - - + self.logger.info(f"Processing file: {self.writer.image_filename}") self.logger.info("Step 1/5: Image Enhancement") img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) From 90f1d7aa47e481731e0ec021f9af070b8bf9a0fd Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 12:10:11 +0200 Subject: [PATCH 201/374] rm summary msg (info already logged elsewhere) --- src/eynollah/eynollah.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 82073c3..ed2c9fb 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5172,19 +5172,6 @@ class Eynollah: all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_lines_xml, ocr_all_textlines, ocr_all_textlines_h, ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, ocr_all_textlines_drop, conf_contours_textregions, conf_contours_textregions_h) - summary = [ - f"Total processing time: {time.time() - t0:.1f}s", - f"Output file: {self.writer.output_filename}" - ] - - if self.ocr: - summary.append("OCR processing completed") - if self.full_layout: - summary.append("Full layout analysis completed") - if self.tables: - summary.append("Table detection completed") - self.logger.info(f"Summary: {summary}") - return pcgts contours_only_text_parent_h = None From 5bd318e6576858718f1953749cb448bd4a7dece0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 12:14:32 +0200 Subject: [PATCH 202/374] rm print statement (already log msg) --- src/eynollah/eynollah.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index ed2c9fb..27277ee 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4555,7 +4555,6 @@ class Eynollah: raise ValueError("run requires either a single image filename or a directory") for img_filename in self.ls_imgs: - print(img_filename, 'img_filename') self.logger.info(img_filename) t0 = time.time() From b75ca0d31fc4b8b2806569aebfa38e3203a0e7a0 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 24 Sep 2025 16:29:05 +0200 Subject: [PATCH 203/374] mb_ro_on_layout: remove copy-pasta code not actually used --- src/eynollah/mb_ro_on_layout.py | 333 +------------------------------- 1 file changed, 3 insertions(+), 330 deletions(-) diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index c03d831..c6c02cf 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -3,46 +3,26 @@ Image enhancer. The output can be written as same scale of input or in new predi """ from logging import Logger -from difflib import SequenceMatcher as sq -from PIL import Image, ImageDraw, ImageFont -import math import os -import sys import time from typing import Optional import atexit -import warnings from functools import partial from pathlib import Path from multiprocessing import cpu_count -import gc -import copy from loky import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 import numpy as np -from ocrd import OcrdPage -from ocrd_utils import getLogger, tf_disable_interactive_logs +from ocrd_utils import getLogger import statistics +import tensorflow as tf from tensorflow.keras.models import load_model from .utils.resize import resize_image -from .utils import ( - crop_image_inside_box -) from .utils.contour import ( - filter_contours_area_of_image, - filter_contours_area_of_image_tables, - find_contours_mean_y_diff, find_new_features_of_contours, - find_features_of_contours, - get_text_region_boxes_by_given_contours, - get_textregion_contours_in_org_image, - get_textregion_contours_in_org_image_light, return_contours_of_image, - return_contours_of_interested_region, - return_contours_of_interested_region_by_min_size, - return_contours_of_interested_textline, return_parent_contours, ) @@ -64,7 +44,7 @@ class machine_based_reading_order_on_layout: self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) atexit.register(self.executor.shutdown) self.dir_models = dir_models - self.model_reading_order_dir = dir_models + "/model_eynollah_reading_order_20250824"#"/model_ens_reading_order_machine_based" + self.model_reading_order_dir = dir_models + "/model_eynollah_reading_order_20250824" try: for device in tf.config.list_physical_devices('GPU'): @@ -76,43 +56,7 @@ class machine_based_reading_order_on_layout: self.light_version = True - def cache_images(self, image_filename=None, image_pil=None, dpi=None): - ret = {} - t_c0 = time.time() - if image_filename: - ret['img'] = cv2.imread(image_filename) - if self.light_version: - self.dpi = 100 - else: - self.dpi = 0#check_dpi(image_filename) - else: - ret['img'] = pil2cv(image_pil) - if self.light_version: - self.dpi = 100 - else: - self.dpi = 0#check_dpi(image_pil) - ret['img_grayscale'] = cv2.cvtColor(ret['img'], cv2.COLOR_BGR2GRAY) - for prefix in ('', '_grayscale'): - ret[f'img{prefix}_uint8'] = ret[f'img{prefix}'].astype(np.uint8) - self._imgs = ret - if dpi is not None: - self.dpi = dpi - def reset_file_name_dir(self, image_filename): - t_c = time.time() - self.cache_images(image_filename=image_filename) - self.output_filename = os.path.join(self.dir_out, Path(image_filename).stem +'.png') - - def imread(self, grayscale=False, uint8=True): - key = 'img' - if grayscale: - key += '_grayscale' - if uint8: - key += '_uint8' - return self._imgs[key].copy() - - def isNaN(self, num): - return num != num @staticmethod def our_load_model(model_file): @@ -126,278 +70,7 @@ class machine_based_reading_order_on_layout: "PatchEncoder": PatchEncoder, "Patches": Patches}) return model - def predict_enhancement(self, img): - self.logger.debug("enter predict_enhancement") - - img_height_model = self.model_enhancement.layers[-1].output_shape[1] - img_width_model = self.model_enhancement.layers[-1].output_shape[2] - if img.shape[0] < img_height_model: - img = cv2.resize(img, (img.shape[1], img_width_model), interpolation=cv2.INTER_NEAREST) - if img.shape[1] < img_width_model: - img = cv2.resize(img, (img_height_model, img.shape[0]), interpolation=cv2.INTER_NEAREST) - margin = int(0.1 * img_width_model) - width_mid = img_width_model - 2 * margin - height_mid = img_height_model - 2 * margin - img = img / 255. - img_h = img.shape[0] - img_w = img.shape[1] - - prediction_true = np.zeros((img_h, img_w, 3)) - nxf = img_w / float(width_mid) - nyf = img_h / float(height_mid) - nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) - nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) - - for i in range(nxf): - for j in range(nyf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - else: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - if j == 0: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - else: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - - if index_x_u > img_w: - index_x_u = img_w - index_x_d = img_w - img_width_model - if index_y_u > img_h: - index_y_u = img_h - index_y_d = img_h - img_height_model - - img_patch = img[np.newaxis, index_y_d:index_y_u, index_x_d:index_x_u, :] - label_p_pred = self.model_enhancement.predict(img_patch, verbose=0) - seg = label_p_pred[0, :, :, :] * 255 - - if i == 0 and j == 0: - prediction_true[index_y_d + 0:index_y_u - margin, - index_x_d + 0:index_x_u - margin] = \ - seg[0:-margin or None, - 0:-margin or None] - elif i == nxf - 1 and j == nyf - 1: - prediction_true[index_y_d + margin:index_y_u - 0, - index_x_d + margin:index_x_u - 0] = \ - seg[margin:, - margin:] - elif i == 0 and j == nyf - 1: - prediction_true[index_y_d + margin:index_y_u - 0, - index_x_d + 0:index_x_u - margin] = \ - seg[margin:, - 0:-margin or None] - elif i == nxf - 1 and j == 0: - prediction_true[index_y_d + 0:index_y_u - margin, - index_x_d + margin:index_x_u - 0] = \ - seg[0:-margin or None, - margin:] - elif i == 0 and j != 0 and j != nyf - 1: - prediction_true[index_y_d + margin:index_y_u - margin, - index_x_d + 0:index_x_u - margin] = \ - seg[margin:-margin or None, - 0:-margin or None] - elif i == nxf - 1 and j != 0 and j != nyf - 1: - prediction_true[index_y_d + margin:index_y_u - margin, - index_x_d + margin:index_x_u - 0] = \ - seg[margin:-margin or None, - margin:] - elif i != 0 and i != nxf - 1 and j == 0: - prediction_true[index_y_d + 0:index_y_u - margin, - index_x_d + margin:index_x_u - margin] = \ - seg[0:-margin or None, - margin:-margin or None] - elif i != 0 and i != nxf - 1 and j == nyf - 1: - prediction_true[index_y_d + margin:index_y_u - 0, - index_x_d + margin:index_x_u - margin] = \ - seg[margin:, - margin:-margin or None] - else: - prediction_true[index_y_d + margin:index_y_u - margin, - index_x_d + margin:index_x_u - margin] = \ - seg[margin:-margin or None, - margin:-margin or None] - - prediction_true = prediction_true.astype(int) - return prediction_true - def calculate_width_height_by_columns(self, img, num_col, width_early, label_p_pred): - self.logger.debug("enter calculate_width_height_by_columns") - if num_col == 1: - img_w_new = 2000 - elif num_col == 2: - img_w_new = 2400 - elif num_col == 3: - img_w_new = 3000 - elif num_col == 4: - img_w_new = 4000 - elif num_col == 5: - img_w_new = 5000 - elif num_col == 6: - img_w_new = 6500 - else: - img_w_new = width_early - img_h_new = img_w_new * img.shape[0] // img.shape[1] - - if img_h_new >= 8000: - img_new = np.copy(img) - num_column_is_classified = False - else: - img_new = resize_image(img, img_h_new, img_w_new) - num_column_is_classified = True - - return img_new, num_column_is_classified - - def early_page_for_num_of_column_classification(self,img_bin): - self.logger.debug("enter early_page_for_num_of_column_classification") - if self.input_binary: - img = np.copy(img_bin).astype(np.uint8) - else: - img = self.imread() - img = cv2.GaussianBlur(img, (5, 5), 0) - img_page_prediction = self.do_prediction(False, img, self.model_page) - - imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - thresh = cv2.dilate(thresh, KERNEL, iterations=3) - contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - if len(contours)>0: - cnt_size = np.array([cv2.contourArea(contours[j]) - for j in range(len(contours))]) - cnt = contours[np.argmax(cnt_size)] - box = cv2.boundingRect(cnt) - else: - box = [0, 0, img.shape[1], img.shape[0]] - cropped_page, page_coord = crop_image_inside_box(box, img) - - self.logger.debug("exit early_page_for_num_of_column_classification") - return cropped_page, page_coord - - def calculate_width_height_by_columns_1_2(self, img, num_col, width_early, label_p_pred): - self.logger.debug("enter calculate_width_height_by_columns") - if num_col == 1: - img_w_new = 1000 - else: - img_w_new = 1300 - img_h_new = img_w_new * img.shape[0] // img.shape[1] - - if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: - img_new = np.copy(img) - num_column_is_classified = False - #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: - elif img_h_new >= 8000: - img_new = np.copy(img) - num_column_is_classified = False - else: - img_new = resize_image(img, img_h_new, img_w_new) - num_column_is_classified = True - - return img_new, num_column_is_classified - - def resize_and_enhance_image_with_column_classifier(self, light_version): - self.logger.debug("enter resize_and_enhance_image_with_column_classifier") - dpi = 0#self.dpi - self.logger.info("Detected %s DPI", dpi) - if self.input_binary: - img = self.imread() - prediction_bin = self.do_prediction(True, img, self.model_bin, n_batch_inference=5) - prediction_bin = 255 * (prediction_bin[:,:,0]==0) - prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2).astype(np.uint8) - img= np.copy(prediction_bin) - img_bin = prediction_bin - else: - img = self.imread() - self.h_org, self.w_org = img.shape[:2] - img_bin = None - - width_early = img.shape[1] - t1 = time.time() - _, page_coord = self.early_page_for_num_of_column_classification(img_bin) - - self.image_page_org_size = img[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3], :] - self.page_coord = page_coord - - if self.num_col_upper and not self.num_col_lower: - num_col = self.num_col_upper - label_p_pred = [np.ones(6)] - elif self.num_col_lower and not self.num_col_upper: - num_col = self.num_col_lower - label_p_pred = [np.ones(6)] - elif not self.num_col_upper and not self.num_col_lower: - if self.input_binary: - img_in = np.copy(img) - img_in = img_in / 255.0 - img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = img_in.reshape(1, 448, 448, 3) - else: - img_1ch = self.imread(grayscale=True) - width_early = img_1ch.shape[1] - img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - img_1ch = img_1ch / 255.0 - img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) - img_in[0, :, :, 0] = img_1ch[:, :] - img_in[0, :, :, 1] = img_1ch[:, :] - img_in[0, :, :, 2] = img_1ch[:, :] - - label_p_pred = self.model_classifier.predict(img_in, verbose=0) - num_col = np.argmax(label_p_pred[0]) + 1 - elif (self.num_col_upper and self.num_col_lower) and (self.num_col_upper!=self.num_col_lower): - if self.input_binary: - img_in = np.copy(img) - img_in = img_in / 255.0 - img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = img_in.reshape(1, 448, 448, 3) - else: - img_1ch = self.imread(grayscale=True) - width_early = img_1ch.shape[1] - img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - img_1ch = img_1ch / 255.0 - img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) - img_in[0, :, :, 0] = img_1ch[:, :] - img_in[0, :, :, 1] = img_1ch[:, :] - img_in[0, :, :, 2] = img_1ch[:, :] - - label_p_pred = self.model_classifier.predict(img_in, verbose=0) - num_col = np.argmax(label_p_pred[0]) + 1 - - if num_col > self.num_col_upper: - num_col = self.num_col_upper - label_p_pred = [np.ones(6)] - if num_col < self.num_col_lower: - num_col = self.num_col_lower - label_p_pred = [np.ones(6)] - else: - num_col = self.num_col_upper - label_p_pred = [np.ones(6)] - - self.logger.info("Found %d columns (%s)", num_col, np.around(label_p_pred, decimals=5)) - - if dpi < DPI_THRESHOLD: - if light_version and num_col in (1,2): - img_new, num_column_is_classified = self.calculate_width_height_by_columns_1_2( - img, num_col, width_early, label_p_pred) - else: - img_new, num_column_is_classified = self.calculate_width_height_by_columns( - img, num_col, width_early, label_p_pred) - if light_version: - image_res = np.copy(img_new) - else: - image_res = self.predict_enhancement(img_new) - is_image_enhanced = True - - else: - num_column_is_classified = True - image_res = np.copy(img) - is_image_enhanced = False - - self.logger.debug("exit resize_and_enhance_image_with_column_classifier") - return is_image_enhanced, img, image_res, num_col, num_column_is_classified, img_bin def read_xml(self, xml_file): file_name = Path(xml_file).stem tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) From c8ebe84697bd20568320526163933840123238b3 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 24 Sep 2025 16:36:18 +0200 Subject: [PATCH 204/374] image_enhancer: add missing models, remove dead code --- src/eynollah/image_enhancer.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index 983712d..7383b91 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -3,29 +3,23 @@ Image enhancer. The output can be written as same scale of input or in new predi """ from logging import Logger -from difflib import SequenceMatcher as sq -from PIL import Image, ImageDraw, ImageFont -import math import os -import sys import time from typing import Optional import atexit -import warnings from functools import partial from pathlib import Path from multiprocessing import cpu_count import gc -import copy from loky import ProcessPoolExecutor -import xml.etree.ElementTree as ET import cv2 import numpy as np -from ocrd import OcrdPage from ocrd_utils import getLogger, tf_disable_interactive_logs -import statistics +import tensorflow as tf +from skimage.morphology import skeletonize from tensorflow.keras.models import load_model from .utils.resize import resize_image +from .utils.pil_cv2 import pil2cv from .utils import ( crop_image_inside_box ) @@ -62,6 +56,7 @@ class Enhancer: self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) atexit.register(self.executor.shutdown) self.dir_models = dir_models + self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" self.model_dir_of_col_classifier = dir_models + "/eynollah-column-classifier_20210425" self.model_page_dir = dir_models + "/eynollah-page-extraction_20210425" @@ -75,10 +70,10 @@ class Enhancer: self.model_page = self.our_load_model(self.model_page_dir) self.model_classifier = self.our_load_model(self.model_dir_of_col_classifier) self.model_enhancement = self.our_load_model(self.model_dir_of_enhancement) + self.model_bin = self.our_load_model(self.model_dir_of_binarization) def cache_images(self, image_filename=None, image_pil=None, dpi=None): ret = {} - t_c0 = time.time() if image_filename: ret['img'] = cv2.imread(image_filename) if self.light_version: @@ -99,7 +94,6 @@ class Enhancer: self.dpi = dpi def reset_file_name_dir(self, image_filename): - t_c = time.time() self.cache_images(image_filename=image_filename) self.output_filename = os.path.join(self.dir_out, Path(image_filename).stem +'.png') From 8b30bdbae2ad630b6f07cc0e82aa67cb01da3e50 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 24 Sep 2025 16:39:31 +0200 Subject: [PATCH 205/374] image_enhancer: use latest page extraction model --- src/eynollah/image_enhancer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index 7383b91..f577e52 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -59,7 +59,7 @@ class Enhancer: self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" self.model_dir_of_col_classifier = dir_models + "/eynollah-column-classifier_20210425" - self.model_page_dir = dir_models + "/eynollah-page-extraction_20210425" + self.model_page_dir = dir_models + "/model_eynollah_page_extraction_20250915" try: for device in tf.config.list_physical_devices('GPU'): From ce13d8c5a329ebc6b6a32464a029801456320548 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 24 Sep 2025 01:22:07 +0200 Subject: [PATCH 206/374] get textlines inside textregion sorted --- src/eynollah/eynollah.py | 44 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 27277ee..93d1c8d 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1753,7 +1753,45 @@ class Eynollah: prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) self.logger.debug("exit extract_text_regions") return prediction_regions, prediction_regions2 + + def get_textlines_of_a_textregion_sorted(self, textlines_textregion, cx_textline, cy_textline): + N = len(cy_textline) + if N==0: + return [] + + diff_matrix = np.abs(np.subtract.outer(cy_textline, cy_textline)) + + non_zero_diffs = diff_matrix[diff_matrix > 0] + if len(non_zero_diffs) == 0: + mean_y_diff = 0 + else: + mean_y_diff = np.mean(non_zero_diffs) + + row_threshold = mean_y_diff / 2 if mean_y_diff > 0 else 10 + indices_sorted_by_y = sorted(range(N), key=lambda i: cy_textline[i]) + + rows = [] + current_row = [indices_sorted_by_y[0]] + for i in range(1, N): + current_idx = indices_sorted_by_y[i] + prev_idx = current_row[0] + if abs(cy_textline[current_idx] - cy_textline[prev_idx]) <= row_threshold: + current_row.append(current_idx) + else: + rows.append(current_row) + current_row = [current_idx] + rows.append(current_row) + + sorted_textlines = [] + for row in rows: + row_sorted = sorted(row, key=lambda i: cx_textline[i]) + for idx in row_sorted: + sorted_textlines.append(textlines_textregion[idx]) + + return sorted_textlines + + def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): polygons_of_textlines = return_contours_of_interested_region(textline_mask_tot,1,0.00001) @@ -1773,8 +1811,12 @@ class Eynollah: results = np.array(results) indexes_in = args_textlines[results==1] textlines_ins = [polygons_of_textlines[ind] for ind in indexes_in] + cx_textline_in = [cx_main_tot[ind] for ind in indexes_in] + cy_textline_in = [cy_main_tot[ind] for ind in indexes_in] - all_found_textline_polygons.append(textlines_ins[::-1]) + textlines_ins = self.get_textlines_of_a_textregion_sorted(textlines_ins, cx_textline_in, cy_textline_in) + + all_found_textline_polygons.append(textlines_ins)#[::-1]) slopes.append(slope_deskew) _, crop_coor = crop_image_inside_box(boxes[index],image_page_rotated) From 6904a981828d70024ce5a97ca2823ea22ac581ad Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 24 Sep 2025 01:25:57 +0200 Subject: [PATCH 207/374] get textlines inside textregion sorted debugging --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 93d1c8d..9acae80 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1789,7 +1789,7 @@ class Eynollah: for idx in row_sorted: sorted_textlines.append(textlines_textregion[idx]) - return sorted_textlines + return sorted_textlines def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): From 6d8641a518ae9aa1934a95094413ff65c542b986 Mon Sep 17 00:00:00 2001 From: b-vr103 Date: Wed, 24 Sep 2025 03:43:36 +0200 Subject: [PATCH 208/374] get textlines sorted in textregion - verticals --- src/eynollah/eynollah.py | 74 +++++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 23 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9acae80..bbe80fe 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1759,35 +1759,63 @@ class Eynollah: if N==0: return [] - diff_matrix = np.abs(np.subtract.outer(cy_textline, cy_textline)) + diff_cy = np.abs( np.diff(sorted(cy_textline)) ) + diff_cx = np.abs(np.diff(sorted(cx_textline)) ) + - non_zero_diffs = diff_matrix[diff_matrix > 0] - if len(non_zero_diffs) == 0: - mean_y_diff = 0 + if len(diff_cy)>0: + mean_y_diff = np.mean(diff_cy) + mean_x_diff = np.mean(diff_cx) else: - mean_y_diff = np.mean(non_zero_diffs) + mean_y_diff = 0 + mean_x_diff = 0 - row_threshold = mean_y_diff / 2 if mean_y_diff > 0 else 10 - indices_sorted_by_y = sorted(range(N), key=lambda i: cy_textline[i]) + if np.int(mean_y_diff) >= np.int(mean_x_diff): + row_threshold = mean_y_diff / 2 if mean_y_diff > 0 else 10 + + indices_sorted_by_y = sorted(range(N), key=lambda i: cy_textline[i]) - rows = [] - current_row = [indices_sorted_by_y[0]] - for i in range(1, N): - current_idx = indices_sorted_by_y[i] - prev_idx = current_row[0] - if abs(cy_textline[current_idx] - cy_textline[prev_idx]) <= row_threshold: - current_row.append(current_idx) - else: - rows.append(current_row) - current_row = [current_idx] - rows.append(current_row) + rows = [] + current_row = [indices_sorted_by_y[0]] + for i in range(1, N): + current_idx = indices_sorted_by_y[i] + prev_idx = current_row[0] + if abs(cy_textline[current_idx] - cy_textline[prev_idx]) <= row_threshold: + current_row.append(current_idx) + else: + rows.append(current_row) + current_row = [current_idx] + rows.append(current_row) - sorted_textlines = [] - for row in rows: - row_sorted = sorted(row, key=lambda i: cx_textline[i]) - for idx in row_sorted: - sorted_textlines.append(textlines_textregion[idx]) + sorted_textlines = [] + for row in rows: + row_sorted = sorted(row, key=lambda i: cx_textline[i]) + for idx in row_sorted: + sorted_textlines.append(textlines_textregion[idx]) + + else: + row_threshold = mean_x_diff / 2 if mean_x_diff > 0 else 10 + indices_sorted_by_x = sorted(range(N), key=lambda i: cx_textline[i]) + + rows = [] + current_row = [indices_sorted_by_x[0]] + + for i in range(1, N): + current_idy = indices_sorted_by_x[i] + prev_idy = current_row[0] + if abs(cx_textline[current_idy] - cx_textline[prev_idy] ) <= row_threshold: + current_row.append(current_idy) + else: + rows.append(current_row) + current_row = [current_idy] + rows.append(current_row) + + sorted_textlines = [] + for row in rows: + row_sorted = sorted(row , key=lambda i: cy_textline[i]) + for idy in row_sorted: + sorted_textlines.append(textlines_textregion[idy]) return sorted_textlines From 80d50d4bf6e7c1dfc211bc802728137ffd9f2ee6 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 24 Sep 2025 16:36:00 +0200 Subject: [PATCH 209/374] get textlines sorted in textregion - verticals --- src/eynollah/eynollah.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index bbe80fe..6b5b74e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1772,7 +1772,7 @@ class Eynollah: if np.int(mean_y_diff) >= np.int(mean_x_diff): - row_threshold = mean_y_diff / 2 if mean_y_diff > 0 else 10 + row_threshold = mean_y_diff / 1.5 if mean_y_diff > 0 else 10 indices_sorted_by_y = sorted(range(N), key=lambda i: cy_textline[i]) @@ -1795,7 +1795,7 @@ class Eynollah: sorted_textlines.append(textlines_textregion[idx]) else: - row_threshold = mean_x_diff / 2 if mean_x_diff > 0 else 10 + row_threshold = mean_x_diff / 1.5 if mean_x_diff > 0 else 10 indices_sorted_by_x = sorted(range(N), key=lambda i: cx_textline[i]) rows = [] @@ -4693,7 +4693,12 @@ class Eynollah: all_found_textline_polygons = filter_contours_area_of_image( textline_mask_tot_ea, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001) - all_found_textline_polygons = all_found_textline_polygons[::-1] + M_main_tot = [cv2.moments(all_found_textline_polygons[j]) + for j in range(len(all_found_textline_polygons))] + cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] + cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] + + all_found_textline_polygons = self.get_textlines_of_a_textregion_sorted(all_found_textline_polygons, cx_main_tot, cy_main_tot)#all_found_textline_polygons[::-1] all_found_textline_polygons=[ all_found_textline_polygons ] From 960b11f51f98518feaa5b1989a71bc368e6c9fa4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 22:58:57 +0200 Subject: [PATCH 210/374] machine-based-reading-order CLI: no foreign logger, add --log-level --- src/eynollah/cli.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 1170465..420373a 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -37,14 +37,22 @@ def main(): type=click.Path(exists=True, file_okay=False), required=True, ) +@click.option( + "--log_level", + "-l", + type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), + help="Override log level globally to this", +) + +def machine_based_reading_order(dir_xml, xml_file, dir_out, model, log_level): + orderer = machine_based_reading_order_on_layout(model, dir_out=dir_out) + if log_level: + orderer.logger.setLevel(getLevelName(log_level)) -def machine_based_reading_order(dir_xml, xml_file, dir_out, model): - raedingorder_object = machine_based_reading_order_on_layout(model, dir_out=dir_out, logger=getLogger('enhancement')) - if dir_xml: - raedingorder_object.run(dir_in=dir_xml) + orderer.run(dir_in=dir_xml) else: - raedingorder_object.run(xml_filename=xml_file) + orderer.run(xml_filename=xml_file) @main.command() From 8a1e5a895057aac0d0dd878e58c0ce3e70c891fe Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 23:03:11 +0200 Subject: [PATCH 211/374] enhancement / layout CLI: do not override logger name --- src/eynollah/cli.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 420373a..ab157d1 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -137,21 +137,20 @@ def binarization(patches, model_dir, input_image, dir_in, output): def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_lower, save_org_scale, log_level): initLogging() - if log_level: - getLogger('enhancement').setLevel(getLevelName(log_level)) assert image or dir_in, "Either a single image -i or a dir_in -di is required" - enhancer_object = Enhancer( + enhancer = Enhancer( model, - logger=getLogger('enhancement'), dir_out=out, num_col_upper=num_col_upper, num_col_lower=num_col_lower, save_org_scale=save_org_scale, ) + if log_level: + enhancer.logger.setLevel(getLevelName(log_level)) if dir_in: - enhancer_object.run(dir_in=dir_in, overwrite=overwrite) + enhancer.run(dir_in=dir_in, overwrite=overwrite) else: - enhancer_object.run(image_filename=image, overwrite=overwrite) + enhancer.run(image_filename=image, overwrite=overwrite) @main.command() @click.option( @@ -368,8 +367,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ getLogger('eynollah').setLevel(logging.INFO) else: initLogging() - if log_level: - getLogger('eynollah').setLevel(getLevelName(log_level)) assert enable_plotting or not save_layout, "Plotting with -sl also requires -ep" assert enable_plotting or not save_deskewed, "Plotting with -sd also requires -ep" assert enable_plotting or not save_all, "Plotting with -sa also requires -ep" @@ -420,6 +417,8 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ threshold_art_class_textline=threshold_art_class_textline, threshold_art_class_layout=threshold_art_class_layout, ) + if log_level: + eynollah.logger.setLevel(getLevelName(log_level)) if dir_in: eynollah.run(dir_in=dir_in, overwrite=overwrite) else: @@ -529,8 +528,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, model_name, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): initLogging() - if log_level: - getLogger('eynollah').setLevel(getLevelName(log_level)) assert not model or not model_name, "model directory -m can not be set alongside specific model name --model_name" assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text -etit can not be set alongside transformer ocr -tr_ocr" @@ -557,6 +554,8 @@ def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, pref_of_dataset=dataset_abbrevation, min_conf_value_of_textline_text=min_conf_value_of_textline_text, ) + if log_level: + eynollah_ocr.logger.setLevel(getLevelName(log_level)) eynollah_ocr.run(overwrite=overwrite) if __name__ == "__main__": From 93f7588bfa3787679fd5bb843176ea453c303f44 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 23:08:50 +0200 Subject: [PATCH 212/374] binarizer CLI: add --log-level --- src/eynollah/cli.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index ab157d1..19beab5 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -71,10 +71,18 @@ def machine_based_reading_order(dir_xml, xml_file, dir_out, model, log_level): help="output image (if using -i) or output image directory (if using -di)", type=click.Path(file_okay=True, dir_okay=True), ) -def binarization(patches, model_dir, input_image, dir_in, output): +@click.option( + "--log_level", + "-l", + type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), + help="Override log level globally to this", +) +def binarization(patches, model_dir, input_image, dir_in, output, log_level): assert (dir_in is None) != (input_image is None), "Specify either -di and or -i not both" - SbbBinarizer(model_dir).run(image_path=input_image, use_patches=patches, output=output, dir_in=dir_in) - + binarizer = SbbBinarizer(model_dir) + if log_level: + binarizer.log.setLevel(getLevelName(log_level)) + binarizer.run(image_path=input_image, use_patches=patches, output=output, dir_in=dir_in) @main.command() From 96a0d22496eca2497abac64dcb931d9d45d3394c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 24 Sep 2025 23:52:35 +0200 Subject: [PATCH 213/374] mbreorder CLI: change options to mimic other commands --- src/eynollah/cli.py | 49 +++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 19beab5..71958df 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -13,20 +13,20 @@ def main(): @main.command() @click.option( - "--dir_xml", - "-dx", - help="directory of page-xml files", + "--dir_in", + "-di", + help="directory of PAGE-XML input files", type=click.Path(exists=True, file_okay=False), ) @click.option( - "--xml_file", - "-xml", - help="xml filename", + "--input", + "-i", + help="PAGE-XML input filename", type=click.Path(exists=True, dir_okay=False), ) @click.option( - "--dir_out", - "-do", + "--out", + "-o", help="directory for output images", type=click.Path(exists=True, file_okay=False), ) @@ -44,21 +44,26 @@ def main(): help="Override log level globally to this", ) -def machine_based_reading_order(dir_xml, xml_file, dir_out, model, log_level): - orderer = machine_based_reading_order_on_layout(model, dir_out=dir_out) +def machine_based_reading_order(dir_in, input, out, model, log_level): + orderer = machine_based_reading_order_on_layout(model, dir_out=out) if log_level: orderer.logger.setLevel(getLevelName(log_level)) - if dir_xml: - orderer.run(dir_in=dir_xml) + if dir_in: + orderer.run(dir_in=dir_in) else: - orderer.run(xml_filename=xml_file) + orderer.run(xml_filename=input) @main.command() @click.option('--patches/--no-patches', default=True, help='by enabling this parameter you let the model to see the image in patches.') @click.option('--model_dir', '-m', type=click.Path(exists=True, file_okay=False), required=True, help='directory containing models for prediction') -@click.option("--input-image", "-i", help="input image", type=click.Path(exists=True, dir_okay=False)) +@click.option( + "--input-image", "--image", + "-i", + help="input image filename", + type=click.Path(exists=True, dir_okay=False) +) @click.option( "--dir_in", "-di", @@ -89,14 +94,14 @@ def binarization(patches, model_dir, input_image, dir_in, output, log_level): @click.option( "--image", "-i", - help="image filename", + help="input image filename", type=click.Path(exists=True, dir_okay=False), ) @click.option( "--out", "-o", - help="directory to write output xml data", + help="directory for output PAGE-XML files", type=click.Path(exists=True, file_okay=False), required=True, ) @@ -109,7 +114,7 @@ def binarization(patches, model_dir, input_image, dir_in, output, log_level): @click.option( "--dir_in", "-di", - help="directory of images", + help="directory of input images", type=click.Path(exists=True, file_okay=False), ) @click.option( @@ -164,14 +169,14 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low @click.option( "--image", "-i", - help="image filename", + help="input image filename", type=click.Path(exists=True, dir_okay=False), ) @click.option( "--out", "-o", - help="directory to write output xml data", + help="directory for output PAGE-XML files", type=click.Path(exists=True, file_okay=False), required=True, ) @@ -184,7 +189,7 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low @click.option( "--dir_in", "-di", - help="directory of images", + help="directory of input images", type=click.Path(exists=True, file_okay=False), ) @click.option( @@ -437,7 +442,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ @click.option( "--image", "-i", - help="image filename", + help="input image filename", type=click.Path(exists=True, dir_okay=False), ) @click.option( @@ -449,7 +454,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ @click.option( "--dir_in", "-di", - help="directory of images", + help="directory of input images", type=click.Path(exists=True, file_okay=False), ) @click.option( From d6cdb69acbd1770c080ede18f52ed05c608a3693 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 00:11:23 +0200 Subject: [PATCH 214/374] binarize/enhance/layout/ocr ls_imgs: use the same file name suffix filter for dir-in mode --- src/eynollah/eynollah.py | 11 +++++------ src/eynollah/image_enhancer.py | 7 ++++--- src/eynollah/sbb_binarize.py | 3 ++- src/eynollah/utils/__init__.py | 8 ++++++++ 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 27277ee..9071f7a 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -107,6 +107,7 @@ from .utils.drop_capitals import ( from .utils.marginals import get_marginals from .utils.resize import resize_image from .utils import ( + is_image_filename, boosting_headers_by_longshot_region_segmentation, crop_image_inside_box, find_num_col, @@ -4547,14 +4548,13 @@ class Eynollah: self.logger.info("Enabled modes: " + ", ".join(enabled_modes)) if dir_in: - self.ls_imgs = os.listdir(dir_in) - self.ls_imgs = [ind_img for ind_img in self.ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG') or ind_img.endswith('.TIF') or ind_img.endswith('.TIFF') or ind_img.endswith('.PNG')] + ls_imgs = list(filter(is_image_filename, os.listdir(self.dir_in))) elif image_filename: - self.ls_imgs = [image_filename] + ls_imgs = [image_filename] else: raise ValueError("run requires either a single image filename or a directory") - for img_filename in self.ls_imgs: + for img_filename in ls_imgs: self.logger.info(img_filename) t0 = time.time() @@ -5394,8 +5394,7 @@ class Eynollah_ocr: def run(self, overwrite : bool = False): if self.dir_in: - ls_imgs = os.listdir(self.dir_in) - ls_imgs = [ind_img for ind_img in ls_imgs if ind_img.endswith('.jpg') or ind_img.endswith('.jpeg') or ind_img.endswith('.png') or ind_img.endswith('.tif') or ind_img.endswith('.tiff') or ind_img.endswith('.JPG') or ind_img.endswith('.JPEG') or ind_img.endswith('.TIF') or ind_img.endswith('.TIFF') or ind_img.endswith('.PNG')] + ls_imgs = list(filter(is_image_filename, os.listdir(self.dir_in))) else: ls_imgs = [self.image_filename] diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index f577e52..5a06d59 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -21,6 +21,7 @@ from tensorflow.keras.models import load_model from .utils.resize import resize_image from .utils.pil_cv2 import pil2cv from .utils import ( + is_image_filename, crop_image_inside_box ) @@ -701,13 +702,13 @@ class Enhancer: t0_tot = time.time() if dir_in: - self.ls_imgs = os.listdir(dir_in) + ls_imgs = list(filter(is_image_filename, os.listdir(dir_in))) elif image_filename: - self.ls_imgs = [image_filename] + ls_imgs = [image_filename] else: raise ValueError("run requires either a single image filename or a directory") - for img_filename in self.ls_imgs: + for img_filename in ls_imgs: self.logger.info(img_filename) t0 = time.time() diff --git a/src/eynollah/sbb_binarize.py b/src/eynollah/sbb_binarize.py index 2d5035f..3716987 100644 --- a/src/eynollah/sbb_binarize.py +++ b/src/eynollah/sbb_binarize.py @@ -16,6 +16,7 @@ import tensorflow as tf from tensorflow.keras.models import load_model from tensorflow.python.keras import backend as tensorflow_backend +from .utils import is_image_filename def resize_image(img_in, input_height, input_width): return cv2.resize(img_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) @@ -347,7 +348,7 @@ class SbbBinarizer: cv2.imwrite(output, img_last) return img_last else: - ls_imgs = os.listdir(dir_in) + ls_imgs = list(filter(is_image_filename, os.listdir(dir_in))) for image_name in ls_imgs: image_stem = image_name.split('.')[0] print(image_name,'image_name') diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index ca86047..c154fe4 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -2194,3 +2194,11 @@ def return_boxes_of_images_by_order_of_reading_new( return boxes, peaks_neg_tot_tables_new else: return boxes, peaks_neg_tot_tables + +def is_image_filename(fname: str) -> bool: + return fname.lower().endswith(('.jpg', + '.jpeg', + '.png', + '.tif', + '.tiff', + )) From b094a6b77feb4e86f1ae07c1a5c96e5e88068523 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 00:51:45 +0200 Subject: [PATCH 215/374] mbreorder: avoid spaces in logger name --- src/eynollah/mb_ro_on_layout.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index c6c02cf..70f1402 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -39,7 +39,7 @@ class machine_based_reading_order_on_layout: ): self.dir_out = dir_out - self.logger = logger if logger else getLogger('mbro on layout') + self.logger = logger if logger else getLogger('mbreorder') # for parallelization of CPU-intensive tasks: self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) atexit.register(self.executor.shutdown) From 9967510327d33a49aa619ceba7a36f414fdc09e7 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 00:52:16 +0200 Subject: [PATCH 216/374] mbreorder: filter by .xml suffix in dir-in mode --- src/eynollah/mb_ro_on_layout.py | 7 ++++--- src/eynollah/utils/__init__.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index 70f1402..6d72614 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -25,6 +25,7 @@ from .utils.contour import ( return_contours_of_image, return_parent_contours, ) +from .utils import is_xml_filename DPI_THRESHOLD = 298 KERNEL = np.ones((5, 5), np.uint8) @@ -751,13 +752,13 @@ class machine_based_reading_order_on_layout: t0_tot = time.time() if dir_in: - self.ls_xmls = os.listdir(dir_in) + ls_xmls = list(filter(is_xml_filename, os.listdir(dir_in))) elif xml_filename: - self.ls_xmls = [xml_filename] + ls_xmls = [xml_filename] else: raise ValueError("run requires either a single image filename or a directory") - for xml_filename in self.ls_xmls: + for xml_filename in ls_xmls: self.logger.info(xml_filename) t0 = time.time() diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index c154fe4..6eeabd0 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -2202,3 +2202,6 @@ def is_image_filename(fname: str) -> bool: '.tif', '.tiff', )) + +def is_xml_filename(fname: str) -> bool: + return fname.lower().endswith('.xml') From f07df080f08d93564eafa96c9d6299f181857fbe Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 01:13:48 +0200 Subject: [PATCH 217/374] add tests for enhancement and mbreorder --- .../euler_rechenkunst01_1738_0025.xml | 1626 +++++++++++++ .../resources/kant_aufklaerung_1784_0020.xml | 2129 +++++++++++++++++ tests/test_run.py | 129 +- 3 files changed, 3875 insertions(+), 9 deletions(-) create mode 100644 tests/resources/euler_rechenkunst01_1738_0025.xml create mode 100644 tests/resources/kant_aufklaerung_1784_0020.xml diff --git a/tests/resources/euler_rechenkunst01_1738_0025.xml b/tests/resources/euler_rechenkunst01_1738_0025.xml new file mode 100644 index 0000000..1a92f73 --- /dev/null +++ b/tests/resources/euler_rechenkunst01_1738_0025.xml @@ -0,0 +1,1626 @@ + + + OCR-D + 2016-09-29T14:32:09 + 2018-04-25T08:56:33 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 9 + + + 9 + + + 9 + + + + + + + + + der + + + + + rechten + + + + + gegen + + + + + der + + + + + lincken + + + + + Hand + + + + + bedeutet + + + der rechten gegen der lincken Hand bedeutet + + + + + + + + wie + + + + + folget: + + + wie folget: + + + der rechten gegen der lincken Hand bedeutet +wie folget: + + + + + + + + + I. + + + I. + + + I. + + + + + + + + + 0 + + + + + - + + + + + nichts + + + 0 - nichts + + + + + + + + 1 + + + + + - + + + + + eins + + + 1 - eins + + + + + + + + 2 + + + + + - + + + + + zwey + + + 2 - zwey + + + + + + + + 3 + + + + + - + + + + + drey + + + 3 - drey + + + + + + + + 4 + + + + + - + + + + + vier + + + 4 - vier + + + 0 - nichts +1 - eins +2 - zwey +3 - drey +4 - vier + + + + + + + + + 5 + + + + + - + + + + + fuͤnf + + + 5 - fuͤnf + + + + + + + + 6 + + + + + - + + + + + ſechs + + + 6 - ſechs + + + + + + + 7 + + + + + - + + + + + ſieben + + + 7 - ſieben + + + + + + + + 8 + + + + + - + + + + + acht + + + 8 - acht + + + + + + + + 9 + + + + + - + + + + + neun + + + 9 - neun + + + 5 - fuͤnf +6 - ſechs +7 - ſieben +8 - acht +9 - neun + + + + + + + + + Auf + + + + + der + + + + + zweyten + + + + + Stelle + + + + + aber + + + + + bedeutet. + + + Auf der zweyten Stelle aber bedeutet. + + + Auf der zweyten Stelle aber bedeutet. + + + + + + + + + II. + + + II. + + + II. + + + + + + + + + 0 + + + + + - + + + + + nichts + + + 0 - nichts + + + + + + + + 1 + + + + + - + + + + + zehen + + + 1 - zehen + + + + + + + + 2 + + + + + - + + + + + zwanzig + + + 2 - zwanzig + + + + + + + 3 + + + + + - + + + + + dreyßig + + + 3 - dreyßig + + + + + + + 4 + + + + + - + + + + + vierzig + + + 4 - vierzig + + 0 - nichts +1 - zehen +2 - zwanzig +3 - dreyßig +4 - vierzig + + + + + + + + + 5 + + + + + - + + + + + fuͤnfzig + + + 5 - fuͤnfzig + + + + + + + + 6 + + + + + - + + + + + ſechzig + + + 6 - ſechzig + + + + + + + 7 + + + + + - + + + + + ſiebenzig + + + 7 - ſiebenzig + + + + + + + 8 + + + + + - + + + + + achtzig + + + 8 - achtzig + + + + + + + 9 + + + + + - + + + + + neunzig + + + 9 - neunzig + + 5 - fuͤnfzig +6 - ſechzig +7 - ſiebenzig +8 - achtzig +9 - neunzig + + + + + + + + + Auf + + + + + der + + + + + dritten + + + + + Stelle + + + + + bedeutet. + + + Auf der dritten Stelle bedeutet. + + + Auf der dritten Stelle bedeutet. + + + + + + + + + III. + + + III. + + + III. + + + + + + + + + 0 + + + + + - + + + + + nichts + + + 0 - nichts + + + + + + + + 1 + + + + + - + + + + + hundert + + + 1 - hundert + + + + + + + + 2 + + + + + - + + + + + zwey + + + + + hundert + + + 2 - zwey hundert + + + + + + + + 3 + + + + + - + + + + + drey + + + + + hundert + + + 3 - drey hundert + + + + + + + + 4 + + + + + - + + + + + vier + + + + + hundert + + + 4 - vier hundert + + + 0 - nichts +1 - hundert +2 - zwey hundert +3 - drey hundert +4 - vier hundert + + + + + + + + + 5 + + + + + - + + + + + fuͤnf + + + + + hundert + + + 5 - fuͤnf hundert + + + + + + + + 6 + + + + + - + + + + + ſechs + + + + + hundert + + + 6 - ſechs hundert + + + + + + + 7 + + + + + - + + + + + ſieben + + + + + hundert + + + 7 - ſieben hundert + + + + + + + + 8 + + + + + - + + + + + acht + + + + + hundert + + + 8 - acht hundert + + + + + + + 9 + + + + + - + + + + + neun + + + + + hundert + + + 9 - neun hundert + + + 5 - fuͤnf hundert +6 - ſechs hundert +7 - ſieben hundert +8 - acht hundert +9 - neun hundert + + + + + + + + + Auf + + + + + der + + + + + vierten + + + + + Stelle + + + + + bedeutet. + + + Auf der vierten Stelle bedeutet. + + + Auf der vierten Stelle bedeutet. + + + + + + + + + IV. + + + IV. + + + IV. + + + + + + + + + 0 + + + + + - + + + + + nichts + + + 0 - nichts + + + + + + + + 1 + + + + + - + + + + + tauſend + + + 1 - tauſend + + + + + + + + 2 + + + + + - + + + + + zwey + + + + + tauſend + + + 2 - zwey tauſend + + + + + + + + 3 + + + + + - + + + + + drey + + + + + tauſend + + + 3 - drey tauſend + + + + + + + + 4 + + + + + - + + + + + vier + + + + + tauſend + + + 4 - vier tauſend + + + 0 - nichts +1 - tauſend +2 - zwey tauſend +3 - drey tauſend +4 - vier tauſend + + + + + + + + + 5 + + + + + - + + + + + fuͤnf + + + + + tauſend + + + 5 - fuͤnf tauſend + + + + + + + + 6 + + + + + - + + + + + ſechs + + + + + tauſend + + + 6 - ſechs tauſend + + + + + + + 7 + + + + + - + + + + + ſieben + + + + + tauſend + + + 7 - ſieben tauſend + + + + + + + + 8 + + + + + - + + + + + acht + + + + + tauſend + + + 8 - acht tauſend + + + + + + + 9 + + + + + - + + + + + neun + + + + + tauſend + + + 9 - neun tauſend + + 5 - fuͤnf tauſend +6 - ſechs tauſend +7 - ſieben tauſend +8 - acht tauſend +9 - neun tauſend + + + + + + + + + Auf + + + + + der + + + + + fuͤnften + + + + + Stelle + + + + + bedeutet. + + + Auf der fuͤnften Stelle bedeutet. + + + Auf der fuͤnften Stelle bedeutet. + + + + + + + + + V. + + + V. + + + V. + + + + + + + + + 0 + + + + + - + + + + + nichts + + + 0 - nichts + + + + + + + + 1 + + + + + - + + + + + zehen + + + + + tauſend + + + 1 - zehen tauſend + + + + + + + + 2 + + + + + - + + + + + zwanzig + + + + + tauſend + + + 2 - zwanzig tauſend + + + + + + + + 3 + + + + + - + + + + + dreyßig + + + + + tauſend + + + 3 - dreyßig tauſend + + + + + + + + 4 + + + + + - + + + + + vierzig + + + + + tauſend + + + 4 - vierzig tauſend + + + 0 - nichts +1 - zehen tauſend +2 - zwanzig tauſend +3 - dreyßig tauſend +4 - vierzig tauſend + + + + + + + + + 5 + + + + + - + + + + + fuͤnfzig + + + + + tauſend + + + 5 - fuͤnfzig tauſend + + + + + + + + 6 + + + + + - + + + + + ſechzig + + + + + tauſend + + + 6 - ſechzig tauſend + + + + + + + 7 + + + + + - + + + + + ſiebenzig + + + + + tauſend + + + 7 - ſiebenzig tauſend + + + + + + + + 8 + + + + + - + + + + + achtzig + + + + + tauſend + + + 8 - achtzig tauſend + + + + + + + 9 + + + + + - + + + + + neunzig + + + + + tauſend + + + 9 - neunzig tauſend + + + 5 - fuͤnfzig tauſend +6 - ſechzig tauſend +7 - ſiebenzig tauſend +8 - achtzig tauſend +9 - neunzig tauſend + + + + + + + + A + + + + + 5 + + + A 5 + + A 5 + + + + + + + + + Anf + + + Anf + + Anf + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/resources/kant_aufklaerung_1784_0020.xml b/tests/resources/kant_aufklaerung_1784_0020.xml new file mode 100644 index 0000000..47484cd --- /dev/null +++ b/tests/resources/kant_aufklaerung_1784_0020.xml @@ -0,0 +1,2129 @@ + + + OCR-D + 2016-09-20T11:09:27.431+02:00 + 2018-04-24T17:44:49.605+01:00 + + + + + + + + + + + + + + + + + + + + + + + ( + + + + + + + 484 + + + + + + + ) + + + + + ( 484 ) + + + + ( 484 ) + + + + + + + + + + + gewiegelt + + + + + + + worden + + + + + + + ; + + + + + + + ſo + + + + + + + ſchaͤdlich + + + + + + + iſt + + + + + + + es + + + + + + + Vorurtheile + + + + + + + zu + + + + + gewiegelt worden; ſo ſchaͤdlich iſt es Vorurtheile zu + + + + + + + + + + pflanzen + + + + + + + , + + + + + + + weil + + + + + + + ſie + + + + + + + ſich + + + + + + + zuletzt + + + + + + + an + + + + + + + denen + + + + + + + ſelbſt + + + + + + + raͤchen + + + + + + + , + + + + + pflanzen, weil ſie ſich zuletzt an denen ſelbſt raͤchen, + + + + + + + + + + die + + + + + + + , + + + + + + + oder + + + + + + + deren + + + + + + + Vorgaͤnger + + + + + + + , + + + + + + + ihre + + + + + + + Urheber + + + + + + + geweſen + + + + + die, oder deren Vorgaͤnger, ihre Urheber geweſen + + + + + + + + + + ſind + + + + + + + . + + + + + + + Daher + + + + + + + kann + + + + + + + ein + + + + + + + Publikum + + + + + + + nur + + + + + + + langſam + + + + + + + zur + + + + + ſind. Daher kann ein Publikum nur langſam zur + + + + + + + + + + Aufklaͤrung + + + + + + + gelangen + + + + + + + . + + + + + + + Durch + + + + + + + eine + + + + + + + Revolution + + + + + + + wird + + + + + Aufklaͤrung gelangen. Durch eine Revolution wird + + + + + + + + + + vielleicht + + + + + + + wohl + + + + + + + ein + + + + + + + Abfall + + + + + + + von + + + + + + + perſoͤnlichem + + + + + + + Despo- + + + + + vielleicht wohl ein Abfall von perſoͤnlichem Despo- + + + + + + + + + + tism + + + + + + + und + + + + + + + gewinnſuͤchtiger + + + + + + + oder + + + + + + + herrſchſüchtiger + + + + + + + Be + + + + + + + - + + + + + tism und gewinnſuͤchtiger oder herrſchſüchtiger Be- + + + + + + + + + + druͤkkung + + + + + + + , + + + + + + + aber + + + + + + + niemals + + + + + + + wahre + + + + + + + Reform + + + + + + + der + + + + + + + Den + + + + + + + - + + + + + druͤkkung, aber niemals wahre Reform der Den- + + + + + + + + + + kungsart + + + + + + + zu + + + + + + + Stande + + + + + + + kommen + + + + + + + ; + + + + + + + ſondern + + + + + + + neue + + + + + + + Vor + + + + + + + - + + + + + kungsart zu Stande kommen; ſondern neue Vor- + + + + + + + + + + urtheile + + + + + + + werden + + + + + + + , + + + + + + + eben + + + + + + + ſowohl + + + + + + + als + + + + + + + die + + + + + + + alten + + + + + + + , + + + + + + + zum + + + + + urtheile werden, eben ſowohl als die alten, zum + + + + + + + + + + Leitbande + + + + + + + des + + + + + + + gedankenloſen + + + + + + + großen + + + + + + + Haufens + + + + + Leitbande des gedankenloſen großen Haufens + + + + + + + + + + dienen + + + + + + + . + + + + + dienen. + + + + gewiegelt worden; ſo ſchaͤdlich iſt es Vorurtheile zu +pflanzen, weil ſie ſich zuletzt an denen ſelbſt raͤchen, +die, oder deren Vorgaͤnger, ihre Urheber geweſen +ſind. Daher kann ein Publikum nur langſam zur +Aufklaͤrung gelangen. Durch eine Revolution wird +vielleicht wohl ein Abfall von perſoͤnlichem Despo- +tism und gewinnſuͤchtiger oder herrſchſüchtiger Be- +druͤkkung, aber niemals wahre Reform der Den- +kungsart zu Stande kommen; ſondern neue Vor- +urtheile werden, eben ſowohl als die alten, zum +Leitbande des gedankenloſen großen Haufens +dienen. + + + + + + + + + + + Zu + + + + + + + dieſer + + + + + + + Aufklaͤrung + + + + + + + aber + + + + + + + wird + + + + + + + nichts + + + + + + + erfordert + + + + + Zu dieſer Aufklaͤrung aber wird nichts erfordert + + + + + + + + + + als + + + + + + + Freiheit + + + + + + + ; + + + + + + + und + + + + + + + zwar + + + + + + + die + + + + + + + unſchaͤdlichſte + + + + + + + unter + + + + + als Freiheit; und zwar die unſchaͤdlichſte unter + + + + + + + + + allem + + + + + + + , + + + + + + + was + + + + + + + nur + + + + + + + Freiheit + + + + + + + heißen + + + + + + + mag + + + + + + + , + + + + + + + naͤmlich + + + + + + + die + + + + + + + : + + + + + allem, was nur Freiheit heißen mag, naͤmlich die: + + + + + + + + + + von + + + + + + + ſeiner + + + + + + + Vernunft + + + + + + + in + + + + + + + allen + + + + + + + Stuͤkken + + + + + + + oͤffentlichen + + + + + von ſeiner Vernunft in allen Stuͤkken oͤffentlichen + + + + + + + + + + Gebrauch + + + + + + + zu + + + + + + + machen + + + + + + + . + + + + + + + Nun + + + + + + + hoͤre + + + + + + + ich + + + + + + + aber + + + + + + + von + + + + + + + al + + + + + + + - + + + + + Gebrauch zu machen. Nun hoͤre ich aber von al- + + + + + + + + + + len + + + + + + + Seiten + + + + + + + rufen + + + + + + + : + + + + + + + raͤſonnirt + + + + + + + nicht + + + + + + + ! + + + + + + + Der + + + + + + + Offi + + + + + + + - + + + + + len Seiten rufen: raͤſonnirt nicht! Der Offi- + + + + + + + + + + zier + + + + + + + ſagt + + + + + + + : + + + + + + + raͤſonnirt + + + + + + + nicht + + + + + + + , + + + + + + + ſondern + + + + + + + exercirt + + + + + + + ! + + + + + + + Der + + + + + zier ſagt: raͤſonnirt nicht, ſondern exercirt! Der + + + + + + + + + + Finanzrath + + + + + + + : + + + + + + + raͤſonnirt + + + + + + + nicht + + + + + + + , + + + + + + + ſondern + + + + + + + bezahlt + + + + + + + ! + + + + + + + Der + + + + + Finanzrath: raͤſonnirt nicht, ſondern bezahlt! Der + + + + + + + + + + Geiſtliche + + + + + + + : + + + + + + + raͤſonnirt + + + + + + + nicht + + + + + + + , + + + + + + + ſondern + + + + + + + glaubt + + + + + + + ! + + + + + + + ( + + + + + + + Nur + + + + + Geiſtliche: raͤſonnirt nicht, ſondern glaubt! (Nur + + + + + + + + + + ein + + + + + + + einziger + + + + + + + Herr + + + + + + + in + + + + + + + der + + + + + + + Welt + + + + + + + ſagt + + + + + + + : + + + + + + + raͤſonnirt + + + + + + + , + + + + + + + ſo + + + + + ein einziger Herr in der Welt ſagt: raͤſonnirt, ſo + + + + + + + + + + viel + + + + + + + ihr + + + + + + + wollt + + + + + + + , + + + + + + + und + + + + + + + woruͤber + + + + + + + ihr + + + + + + + wollt + + + + + + + ; + + + + + + + aber + + + + + + + ge + + + + + + + - + + + + + viel ihr wollt, und woruͤber ihr wollt; aber ge- + + + + + + + + + + horcht + + + + + + + ! + + + + + + + ) + + + + + + + Hier + + + + + + + iſt + + + + + + + uͤberall + + + + + + + Einſchraͤnkung + + + + + + + der + + + + + + + Frei + + + + + + + - + + + + + horcht!) Hier iſt uͤberall Einſchraͤnkung der Frei- + + + + + + + + + + heit + + + + + + + . + + + + + + + Welche + + + + + + + Einſchraͤnkung + + + + + + + aber + + + + + + + iſt + + + + + + + der + + + + + + + Aufklaͤ + + + + + + + - + + + + + heit. Welche Einſchraͤnkung aber iſt der Aufklaͤ- + + + + + + + + + + rung + + + + + + + hinderlich + + + + + + + ? + + + + + + + welche + + + + + + + nicht + + + + + + + , + + + + + + + ſondern + + + + + + + ihr + + + + + + + wohl + + + + + + + gar + + + + + rung hinderlich? welche nicht, ſondern ihr wohl gar + + + + + + + + + + befoͤrderlich + + + + + + + ? + + + + + + + — + + + + + + + Ich + + + + + + + antworte + + + + + + + : + + + + + + + der + + + + + + + oͤffentliche + + + + + befoͤrderlich? — Ich antworte: der oͤffentliche + + + + + + + + + + Gebrauch + + + + + + + ſeiner + + + + + + + Vernunft + + + + + + + muß + + + + + + + jederzeit + + + + + + + frei + + + + + + + ſein + + + + + + + , + + + + + Gebrauch ſeiner Vernunft muß jederzeit frei ſein, + + + + + + + + + + und + + + + + + + der + + + + + + + allein + + + + + + + kann + + + + + + + Aufklaͤrung + + + + + + + unter + + + + + + + Menſchen + + + + + + + zu + + + + + und der allein kann Aufklaͤrung unter Menſchen zu + + + + + Zu dieſer Aufklaͤrung aber wird nichts erfordert +als Freiheit; und zwar die unſchaͤdlichſte unter +allem, was nur Freiheit heißen mag, naͤmlich die: +von ſeiner Vernunft in allen Stuͤkken oͤffentlichen +Gebrauch zu machen. Nun hoͤre ich aber von al- +len Seiten rufen: raͤſonnirt nicht! Der Offi- +zier ſagt: raͤſonnirt nicht, ſondern exercirt! Der +Finanzrath: raͤſonnirt nicht, ſondern bezahlt! Der +Geiſtliche: raͤſonnirt nicht, ſondern glaubt! (Nur +ein einziger Herr in der Welt ſagt: raͤſonnirt, ſo +viel ihr wollt, und woruͤber ihr wollt; aber ge- +horcht!) Hier iſt uͤberall Einſchraͤnkung der Frei- +heit. Welche Einſchraͤnkung aber iſt der Aufklaͤ- +rung hinderlich? welche nicht, ſondern ihr wohl gar +befoͤrderlich? — Ich antworte: der oͤffentliche +Gebrauch ſeiner Vernunft muß jederzeit frei ſein, +und der allein kann Aufklaͤrung unter Menſchen zu + + + + + + + + + + + Stan + + + + + + + - + + + + + Stan- + + + + + Stan- + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_run.py b/tests/test_run.py index b4e2dbd..370deef 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -2,7 +2,12 @@ from os import environ from pathlib import Path import logging from PIL import Image -from eynollah.cli import layout as layout_cli, binarization as binarization_cli +from eynollah.cli import ( + layout as layout_cli, + binarization as binarization_cli, + enhancement as enhancement_cli, + machine_based_reading_order as mbreorder_cli, +) from click.testing import CliRunner from ocrd_modelfactory import page_from_file from ocrd_models.constants import NAMESPACES as NS @@ -44,8 +49,7 @@ def test_run_eynollah_layout_filename(tmp_path, subtests, pytestconfig, caplog): options=options): with caplog.filtering(only_eynollah): result = runner.invoke(layout_cli, args + options, catch_exceptions=False) - print(result) - assert result.exit_code == 0 + assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert str(infile) in logmsgs assert outfile.exists() @@ -73,8 +77,7 @@ def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): runner = CliRunner() with caplog.filtering(only_eynollah): result = runner.invoke(layout_cli, args) - print(result) - assert result.exit_code == 0 + assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Job done in')]) == 2 assert any(logmsg for logmsg in logmsgs if logmsg.startswith('All jobs done in')) @@ -88,6 +91,8 @@ def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, ca '-i', str(infile), '-o', str(outfile), ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) caplog.set_level(logging.INFO) def only_eynollah(logrec): return logrec.name == 'SbbBinarizer' @@ -100,8 +105,7 @@ def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, ca options=options): with caplog.filtering(only_eynollah): result = runner.invoke(binarization_cli, args + options) - print(result) - assert result.exit_code == 0 + assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert any(True for logmsg in logmsgs if logmsg.startswith('Predicting')) assert outfile.exists() @@ -119,14 +123,121 @@ def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, c '-di', str(indir), '-o', str(outdir), ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) caplog.set_level(logging.INFO) def only_eynollah(logrec): return logrec.name == 'SbbBinarizer' runner = CliRunner() with caplog.filtering(only_eynollah): result = runner.invoke(binarization_cli, args) - print(result) - assert result.exit_code == 0 + assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Predicting')]) == 2 assert len(list(outdir.iterdir())) == 2 + +def test_run_eynollah_enhancement_filename(tmp_path, subtests, pytestconfig, caplog): + infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') + outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') + args = [ + '-m', EYNOLLAH_MODELS, + '-i', str(infile), + '-o', str(outfile.parent), + # subtests write to same location + '--overwrite', + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'enhancement' + runner = CliRunner() + for options in [ + [], # defaults + ["-sos"], + ]: + with subtests.test(#msg="test CLI", + options=options): + with caplog.filtering(only_eynollah): + result = runner.invoke(enhancement_cli, args + options) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + assert any(True for logmsg in logmsgs if logmsg.startswith('Image was enhanced')), logmsgs + assert outfile.exists() + with Image.open(infile) as original_img: + original_size = original_img.size + with Image.open(outfile) as enhanced_img: + enhanced_size = enhanced_img.size + assert (original_size == enhanced_size) == ("-sos" in options) + +def test_run_eynollah_enhancement_directory(tmp_path, subtests, pytestconfig, caplog): + indir = testdir.joinpath('resources') + outdir = tmp_path + args = [ + '-m', EYNOLLAH_MODELS, + '-di', str(indir), + '-o', str(outdir), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'enhancement' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(enhancement_cli, args) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Image was enhanced')]) == 2 + assert len(list(outdir.iterdir())) == 2 + +def test_run_eynollah_mbreorder_filename(tmp_path, subtests, pytestconfig, caplog): + infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.xml') + outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.xml') + args = [ + '-m', EYNOLLAH_MODELS, + '-i', str(infile), + '-o', str(outfile.parent), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'mbreorder' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(mbreorder_cli, args) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + # FIXME: mbreorder has no logging! + #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs + assert outfile.exists() + #in_tree = page_from_file(str(infile)).etree + #in_order = in_tree.xpath("//page:OrderedGroup//@regionRef", namespaces=NS) + out_tree = page_from_file(str(outfile)).etree + out_order = out_tree.xpath("//page:OrderedGroup//@regionRef", namespaces=NS) + #assert len(out_order) >= 2, "result is inaccurate" + #assert in_order != out_order + assert out_order == ['r_1_1', 'r_2_1', 'r_2_2', 'r_2_3'] + +def test_run_eynollah_mbreorder_directory(tmp_path, subtests, pytestconfig, caplog): + indir = testdir.joinpath('resources') + outdir = tmp_path + args = [ + '-m', EYNOLLAH_MODELS, + '-di', str(indir), + '-o', str(outdir), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'mbreorder' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(mbreorder_cli, args) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + # FIXME: mbreorder has no logging! + #assert len([logmsg for logmsg in logmsgs if logmsg.startswith('???')]) == 2 + assert len(list(outdir.iterdir())) == 2 From 369ef573f9efe520455fd0c3ba9eb64b37c2a819 Mon Sep 17 00:00:00 2001 From: b-vr103 Date: Thu, 25 Sep 2025 02:38:22 +0200 Subject: [PATCH 218/374] get textlines sorted in textregions - detection of vertical and horizontal regions improved --- src/eynollah/eynollah.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6b5b74e..f5d7d8b 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1754,7 +1754,7 @@ class Eynollah: self.logger.debug("exit extract_text_regions") return prediction_regions, prediction_regions2 - def get_textlines_of_a_textregion_sorted(self, textlines_textregion, cx_textline, cy_textline): + def get_textlines_of_a_textregion_sorted(self, textlines_textregion, cx_textline, cy_textline, w_h_textline): N = len(cy_textline) if N==0: return [] @@ -1766,12 +1766,17 @@ class Eynollah: if len(diff_cy)>0: mean_y_diff = np.mean(diff_cy) mean_x_diff = np.mean(diff_cx) + count_hor = np.count_nonzero(np.array(w_h_textline) > 1) + count_ver = len(w_h_textline) - count_hor + else: mean_y_diff = 0 mean_x_diff = 0 + count_hor = 1 + count_ver = 0 - if np.int(mean_y_diff) >= np.int(mean_x_diff): + if count_hor >= count_ver: row_threshold = mean_y_diff / 1.5 if mean_y_diff > 0 else 10 indices_sorted_by_y = sorted(range(N), key=lambda i: cy_textline[i]) @@ -1825,6 +1830,8 @@ class Eynollah: polygons_of_textlines = return_contours_of_interested_region(textline_mask_tot,1,0.00001) M_main_tot = [cv2.moments(polygons_of_textlines[j]) for j in range(len(polygons_of_textlines))] + + w_h_textlines = [cv2.boundingRect(polygons_of_textlines[i])[2:] for i in range(len(polygons_of_textlines))] cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] @@ -1841,8 +1848,9 @@ class Eynollah: textlines_ins = [polygons_of_textlines[ind] for ind in indexes_in] cx_textline_in = [cx_main_tot[ind] for ind in indexes_in] cy_textline_in = [cy_main_tot[ind] for ind in indexes_in] + w_h_textlines_in = [w_h_textlines[ind][0] / float(w_h_textlines[ind][1]) for ind in indexes_in] - textlines_ins = self.get_textlines_of_a_textregion_sorted(textlines_ins, cx_textline_in, cy_textline_in) + textlines_ins = self.get_textlines_of_a_textregion_sorted(textlines_ins, cx_textline_in, cy_textline_in, w_h_textlines_in) all_found_textline_polygons.append(textlines_ins)#[::-1]) slopes.append(slope_deskew) @@ -4695,10 +4703,12 @@ class Eynollah: M_main_tot = [cv2.moments(all_found_textline_polygons[j]) for j in range(len(all_found_textline_polygons))] + w_h_textlines = [cv2.boundingRect(all_found_textline_polygons[j])[2:] for j in range(len(all_found_textline_polygons))] + w_h_textlines = [w_h_textlines[j][0] / float(w_h_textlines[j][1]) for j in range(len(w_h_textlines))] cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - all_found_textline_polygons = self.get_textlines_of_a_textregion_sorted(all_found_textline_polygons, cx_main_tot, cy_main_tot)#all_found_textline_polygons[::-1] + all_found_textline_polygons = self.get_textlines_of_a_textregion_sorted(all_found_textline_polygons, cx_main_tot, cy_main_tot, w_h_textlines)#all_found_textline_polygons[::-1] all_found_textline_polygons=[ all_found_textline_polygons ] From 58dd192fad4dedb4161e2ee9a695039c5d4db964 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 16:05:45 +0200 Subject: [PATCH 219/374] smoke-test: also add enhancement and mbreorder here --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 177e87c..f78d7d1 100644 --- a/Makefile +++ b/Makefile @@ -82,13 +82,21 @@ smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/models_eynollah fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$(basename $( Date: Thu, 25 Sep 2025 16:08:40 +0200 Subject: [PATCH 220/374] CLIs: add required=True where missing --- src/eynollah/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 71958df..9744ecb 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -29,6 +29,7 @@ def main(): "-o", help="directory for output images", type=click.Path(exists=True, file_okay=False), + required=True, ) @click.option( "--model", @@ -75,6 +76,7 @@ def machine_based_reading_order(dir_in, input, out, model, log_level): "-o", help="output image (if using -i) or output image directory (if using -di)", type=click.Path(file_okay=True, dir_okay=True), + required=True, ) @click.option( "--log_level", @@ -475,6 +477,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ "-dx", help="directory of xmls", type=click.Path(exists=True, file_okay=False), + required=True, ) @click.option( "--dir_out_image_text", @@ -492,6 +495,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ "--model_name", help="Specific model file path to use for OCR", type=click.Path(exists=True, file_okay=False), + required=True, ) @click.option( "--tr_ocr", From ef1304a764530802b34c54b8e2a53fbe8a6809d9 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 16:11:39 +0200 Subject: [PATCH 221/374] CLIs: reorder options, explain -i vs -di --- src/eynollah/cli.py | 72 +++++++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 39 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 9744ecb..3e9fbe4 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -12,18 +12,18 @@ def main(): pass @main.command() -@click.option( - "--dir_in", - "-di", - help="directory of PAGE-XML input files", - type=click.Path(exists=True, file_okay=False), -) @click.option( "--input", "-i", help="PAGE-XML input filename", type=click.Path(exists=True, dir_okay=False), ) +@click.option( + "--dir_in", + "-di", + help="directory of PAGE-XML input files (instead of --input)", + type=click.Path(exists=True, file_okay=False), +) @click.option( "--out", "-o", @@ -45,7 +45,8 @@ def main(): help="Override log level globally to this", ) -def machine_based_reading_order(dir_in, input, out, model, log_level): +def machine_based_reading_order(input, dir_in, out, model, log_level): + assert bool(input) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." orderer = machine_based_reading_order_on_layout(model, dir_out=out) if log_level: orderer.logger.setLevel(getLevelName(log_level)) @@ -68,7 +69,7 @@ def machine_based_reading_order(dir_in, input, out, model, log_level): @click.option( "--dir_in", "-di", - help="directory of input images", + help="directory of input images (instead of --image)", type=click.Path(exists=True, file_okay=False), ) @click.option( @@ -85,7 +86,7 @@ def machine_based_reading_order(dir_in, input, out, model, log_level): help="Override log level globally to this", ) def binarization(patches, model_dir, input_image, dir_in, output, log_level): - assert (dir_in is None) != (input_image is None), "Specify either -di and or -i not both" + assert bool(input_image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." binarizer = SbbBinarizer(model_dir) if log_level: binarizer.log.setLevel(getLevelName(log_level)) @@ -116,7 +117,7 @@ def binarization(patches, model_dir, input_image, dir_in, output, log_level): @click.option( "--dir_in", "-di", - help="directory of input images", + help="directory of input images (instead of --image)", type=click.Path(exists=True, file_okay=False), ) @click.option( @@ -151,8 +152,8 @@ def binarization(patches, model_dir, input_image, dir_in, output, log_level): ) def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_lower, save_org_scale, log_level): + assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." initLogging() - assert image or dir_in, "Either a single image -i or a dir_in -di is required" enhancer = Enhancer( model, dir_out=out, @@ -191,7 +192,7 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low @click.option( "--dir_in", "-di", - help="directory of input images", + help="directory of input images (instead of --image)", type=click.Path(exists=True, file_okay=False), ) @click.option( @@ -400,7 +401,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ assert not extract_only_images or not tables, "Image extraction -eoi can not be set alongside tables -tab" assert not extract_only_images or not right2left, "Image extraction -eoi can not be set alongside right2left -r2l" assert not extract_only_images or not headers_off, "Image extraction -eoi can not be set alongside headers_off -ho" - assert image or dir_in, "Either a single image -i or a dir_in -di is required" + assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." eynollah = Eynollah( model, dir_out=out, @@ -447,44 +448,44 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="input image filename", type=click.Path(exists=True, dir_okay=False), ) -@click.option( - "--overwrite", - "-O", - help="overwrite (instead of skipping) if output xml exists", - is_flag=True, -) @click.option( "--dir_in", "-di", - help="directory of input images", + help="directory of input images (instead of --image)", type=click.Path(exists=True, file_okay=False), ) @click.option( "--dir_in_bin", "-dib", - help="directory of binarized images. This should be given if you want to do prediction based on both rgb and bin images. And all bin images are png files", + help="directory of binarized images (in addition to --dir_in for RGB images; filename stems must match the RGB image files, with '.png' suffix).\nPerform prediction using both RGB and binary images. (This does not necessarily improve results, however it may be beneficial for certain document images.)", type=click.Path(exists=True, file_okay=False), ) -@click.option( - "--out", - "-o", - help="directory to write output xml data", - type=click.Path(exists=True, file_okay=False), - required=True, -) @click.option( "--dir_xmls", "-dx", - help="directory of xmls", + help="directory of input PAGE-XML files (in addition to --dir_in; filename stems must match the image files, with '.xml' suffix).", + type=click.Path(exists=True, file_okay=False), + required=True, +) +@click.option( + "--out", + "-o", + help="directory for output PAGE-XML files", type=click.Path(exists=True, file_okay=False), required=True, ) @click.option( "--dir_out_image_text", "-doit", - help="directory of images with predicted text", + help="directory for output images, newly rendered with predicted text", type=click.Path(exists=True, file_okay=False), ) +@click.option( + "--overwrite", + "-O", + help="overwrite (instead of skipping) if output xml exists", + is_flag=True, +) @click.option( "--model", "-m", @@ -515,12 +516,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ is_flag=True, help="if this parameter set to true, cropped textline images will not be masked with textline contour.", ) -@click.option( - "--prediction_with_both_of_rgb_and_bin", - "-brb/-nbrb", - is_flag=True, - help="If this parameter is set to True, the prediction will be performed using both RGB and binary images. However, this does not necessarily improve results; it may be beneficial for certain document images.", -) @click.option( "--batch_size", "-bs", @@ -543,7 +538,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ help="Override log level globally to this", ) -def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, model_name, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, prediction_with_both_of_rgb_and_bin, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): +def ocr(image, dir_in, dir_in_bin, dir_xmls, out, dir_out_image_text, overwrite, model, model_name, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): initLogging() assert not model or not model_name, "model directory -m can not be set alongside specific model name --model_name" @@ -552,8 +547,7 @@ def ocr(image, overwrite, dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, assert not export_textline_images_and_text or not batch_size, "Exporting textline and text -etit can not be set alongside batch size -bs" assert not export_textline_images_and_text or not dir_in_bin, "Exporting textline and text -etit can not be set alongside directory of bin images -dib" assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit" - assert not export_textline_images_and_text or not prediction_with_both_of_rgb_and_bin, "Exporting textline and text -etit can not be set alongside prediction with both rgb and bin -brb" - assert (bool(image) ^ bool(dir_in)), "Either -i (single image) or -di (directory) must be provided, but not both." + assert bool(image) != bool(dir_in), "Either -i (single image) or -di (directory) must be provided, but not both." eynollah_ocr = Eynollah_ocr( image_filename=image, dir_xmls=dir_xmls, From 5b1e0c13276db179f74770408fb805f9a7b84d87 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 16:26:31 +0200 Subject: [PATCH 222/374] layout/ocr: make all path options kwargs to run() instead of attributes; ocr: drop redundant prediction_with_both_of_rgb_and_bin in favour of just bool(dir_in_bin) --- src/eynollah/cli.py | 37 +++-- src/eynollah/eynollah.py | 233 ++++++++++++-------------------- src/eynollah/utils/utils_ocr.py | 4 +- 3 files changed, 110 insertions(+), 164 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 3e9fbe4..a0608f9 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -404,13 +404,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." eynollah = Eynollah( model, - dir_out=out, - dir_of_cropped_images=save_images, extract_only_images=extract_only_images, - dir_of_layout=save_layout, - dir_of_deskewed=save_deskewed, - dir_of_all=save_all, - dir_save_page=save_page, enable_plotting=enable_plotting, allow_enhancement=allow_enhancement, curved_line=curved_line, @@ -435,11 +429,16 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ ) if log_level: eynollah.logger.setLevel(getLevelName(log_level)) - if dir_in: - eynollah.run(dir_in=dir_in, overwrite=overwrite) - else: - eynollah.run(image_filename=image, overwrite=overwrite) - + eynollah.run(overwrite=overwrite, + image_filename=image, + dir_in=dir_in, + dir_out=out, + dir_of_cropped_images=save_images, + dir_of_layout=save_layout, + dir_of_deskewed=save_deskewed, + dir_of_all=save_all, + dir_save_page=save_page, + ) @main.command() @click.option( @@ -549,25 +548,25 @@ def ocr(image, dir_in, dir_in_bin, dir_xmls, out, dir_out_image_text, overwrite, assert not export_textline_images_and_text or not dir_out_image_text, "Exporting textline and text -etit can not be set alongside directory of images with predicted text -doit" assert bool(image) != bool(dir_in), "Either -i (single image) or -di (directory) must be provided, but not both." eynollah_ocr = Eynollah_ocr( - image_filename=image, - dir_xmls=dir_xmls, - dir_out_image_text=dir_out_image_text, - dir_in=dir_in, - dir_in_bin=dir_in_bin, - dir_out=out, dir_models=model, model_name=model_name, tr_ocr=tr_ocr, export_textline_images_and_text=export_textline_images_and_text, do_not_mask_with_textline_contour=do_not_mask_with_textline_contour, - prediction_with_both_of_rgb_and_bin=prediction_with_both_of_rgb_and_bin, batch_size=batch_size, pref_of_dataset=dataset_abbrevation, min_conf_value_of_textline_text=min_conf_value_of_textline_text, ) if log_level: eynollah_ocr.logger.setLevel(getLevelName(log_level)) - eynollah_ocr.run(overwrite=overwrite) + eynollah_ocr.run(overwrite=overwrite, + dir_in=dir_in, + dir_in_bin=dir_in_bin, + image_filename=image, + dir_xmls=dir_xmls, + dir_out_image_text=dir_out_image_text, + dir_out=out, + ) if __name__ == "__main__": main() diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9071f7a..533b38f 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -191,13 +191,7 @@ class Eynollah: def __init__( self, dir_models : str, - dir_out : Optional[str] = None, - dir_of_cropped_images : Optional[str] = None, extract_only_images : bool =False, - dir_of_layout : Optional[str] = None, - dir_of_deskewed : Optional[str] = None, - dir_of_all : Optional[str] = None, - dir_save_page : Optional[str] = None, enable_plotting : bool = False, allow_enhancement : bool = False, curved_line : bool = False, @@ -221,18 +215,12 @@ class Eynollah: skip_layout_and_reading_order : bool = False, ): self.logger = getLogger('eynollah') - + self.plotter = None + if skip_layout_and_reading_order: textline_light = True self.light_version = light_version - self.dir_out = dir_out - self.dir_of_all = dir_of_all - self.dir_save_page = dir_save_page self.reading_order_machine_based = reading_order_machine_based - self.dir_of_deskewed = dir_of_deskewed - self.dir_of_deskewed = dir_of_deskewed - self.dir_of_cropped_images=dir_of_cropped_images - self.dir_of_layout=dir_of_layout self.enable_plotting = enable_plotting self.allow_enhancement = allow_enhancement self.curved_line = curved_line @@ -423,21 +411,11 @@ class Eynollah: if dpi is not None: self.dpi = dpi - def reset_file_name_dir(self, image_filename): + def reset_file_name_dir(self, image_filename, dir_out): t_c = time.time() self.cache_images(image_filename=image_filename) - - self.plotter = None if not self.enable_plotting else EynollahPlotter( - dir_out=self.dir_out, - dir_of_all=self.dir_of_all, - dir_save_page=self.dir_save_page, - dir_of_deskewed=self.dir_of_deskewed, - dir_of_cropped_images=self.dir_of_cropped_images, - dir_of_layout=self.dir_of_layout, - image_filename_stem=Path(Path(image_filename).name).stem) - self.writer = EynollahXmlWriter( - dir_out=self.dir_out, + dir_out=dir_out, image_filename=image_filename, curved_line=self.curved_line, textline_light = self.textline_light) @@ -4525,7 +4503,17 @@ class Eynollah: return ordered_left_marginals, ordered_right_marginals, ordered_left_marginals_textline, ordered_right_marginals_textline, ordered_left_marginals_bbox, ordered_right_marginals_bbox, ordered_left_slopes_marginals, ordered_right_slopes_marginals - def run(self, image_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): + def run(self, + overwrite: bool = False, + image_filename: Optional[str] = None, + dir_in: Optional[str] = None, + dir_out: Optional[str] = None, + dir_of_cropped_images: Optional[str] = None, + dir_of_layout: Optional[str] = None, + dir_of_deskewed: Optional[str] = None, + dir_of_all: Optional[str] = None, + dir_save_page: Optional[str] = None, + ): """ Get image and scales, then extract the page of scanned image """ @@ -4546,9 +4534,19 @@ class Eynollah: enabled_modes.append("Table detection") if enabled_modes: self.logger.info("Enabled modes: " + ", ".join(enabled_modes)) + if self.enable_plotting: + self.logger.info("Saving debug plots") + if dir_of_cropped_images: + self.logger.info(f"Saving cropped images to: {dir_of_cropped_images}") + if dir_of_layout: + self.logger.info(f"Saving layout plots to: {dir_of_layout}") + if dir_of_deskewed: + self.logger.info(f"Saving deskewed images to: {dir_of_deskewed}") if dir_in: - ls_imgs = list(filter(is_image_filename, os.listdir(self.dir_in))) + ls_imgs = [os.path.join(dir_in, image_filename) + for image_filename in filter(is_image_filename, + os.listdir(dir_in))] elif image_filename: ls_imgs = [image_filename] else: @@ -4558,7 +4556,15 @@ class Eynollah: self.logger.info(img_filename) t0 = time.time() - self.reset_file_name_dir(os.path.join(dir_in or "", img_filename)) + self.reset_file_name_dir(img_filename, dir_out) + if self.enable_plotting: + self.plotter = EynollahPlotter(dir_out=dir_out, + dir_of_all=dir_of_all, + dir_save_page=dir_save_page, + dir_of_deskewed=dir_of_deskewed, + dir_of_cropped_images=dir_of_cropped_images, + dir_of_layout=dir_of_layout, + image_filename_stem=Path(image_filename).stem) #print("text region early -11 in %.1fs", time.time() - t0) if os.path.exists(self.writer.output_filename): if overwrite: @@ -5151,19 +5157,6 @@ class Eynollah: self.logger.info("Step 5/5: Output Generation") - output_config = [] - if self.enable_plotting: - output_config.append("Saving debug plots") - if self.dir_of_cropped_images: - output_config.append(f"Saving cropped images to: {self.dir_of_cropped_images}") - if self.dir_of_layout: - output_config.append(f"Saving layout plots to: {self.dir_of_layout}") - if self.dir_of_deskewed: - output_config.append(f"Saving deskewed images to: {self.dir_of_deskewed}") - - if output_config: - self.logger.info("Output configuration:\n * %s", "\n * ".join(output_config)) - pcgts = self.writer.build_pagexml_full_layout( contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, @@ -5283,21 +5276,8 @@ class Eynollah: self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") self.logger.info("Step 5/5: Output Generation") - self.logger.info("Generating PAGE-XML output") - if self.enable_plotting: - self.logger.info("Saving debug plots") - - if self.dir_of_cropped_images: - self.logger.info(f"Saving cropped images to: {self.dir_of_cropped_images}") - - if self.dir_of_layout: - self.logger.info(f"Saving layout plots to: {self.dir_of_layout}") - - if self.dir_of_deskewed: - self.logger.info(f"Saving deskewed images to: {self.dir_of_deskewed}") - pcgts = self.writer.build_pagexml_no_full_layout( txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals_left, polygons_of_marginals_right, @@ -5315,32 +5295,19 @@ class Eynollah_ocr: dir_models, model_name=None, dir_xmls=None, - dir_in=None, - image_filename=None, - dir_in_bin=None, - dir_out=None, - dir_out_image_text=None, tr_ocr=False, batch_size=None, export_textline_images_and_text=False, do_not_mask_with_textline_contour=False, - prediction_with_both_of_rgb_and_bin=False, pref_of_dataset=None, min_conf_value_of_textline_text : Optional[float]=None, logger=None, ): - self.dir_in = dir_in - self.image_filename = image_filename - self.dir_in_bin = dir_in_bin - self.dir_out = dir_out - self.dir_xmls = dir_xmls self.dir_models = dir_models self.model_name = model_name self.tr_ocr = tr_ocr self.export_textline_images_and_text = export_textline_images_and_text self.do_not_mask_with_textline_contour = do_not_mask_with_textline_contour - self.dir_out_image_text = dir_out_image_text - self.prediction_with_both_of_rgb_and_bin = prediction_with_both_of_rgb_and_bin self.pref_of_dataset = pref_of_dataset self.logger = logger if logger else getLogger('eynollah') @@ -5392,23 +5359,27 @@ class Eynollah_ocr: ) self.end_character = len(characters) + 2 - def run(self, overwrite : bool = False): - if self.dir_in: - ls_imgs = list(filter(is_image_filename, os.listdir(self.dir_in))) + def run(self, overwrite: bool = False, + dir_in: Optional[str] = None, + dir_in_bin: Optional[str] = None, + image_filename: Optional[str] = None, + dir_xmls: Optional[str] = None, + dir_out_image_text: Optional[str] = None, + dir_out: Optional[str] = None, + ): + if dir_in: + ls_imgs = [os.path.join(dir_in, image_filename) + for image_filename in filter(is_image_filename, + os.listdir(dir_in))] else: - ls_imgs = [self.image_filename] - + ls_imgs = [image_filename] + if self.tr_ocr: tr_ocr_input_height_and_width = 384 - for ind_img in ls_imgs: - if self.dir_in: - file_name = Path(ind_img).stem - dir_img = os.path.join(self.dir_in, ind_img) - else: - file_name = Path(self.image_filename).stem - dir_img = self.image_filename - dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') - out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') + for dir_img in ls_imgs: + file_name = Path(dir_img).stem + dir_xml = os.path.join(dir_xmls, file_name+'.xml') + out_file_ocr = os.path.join(dir_out, file_name+'.xml') if os.path.exists(out_file_ocr): if overwrite: @@ -5419,8 +5390,8 @@ class Eynollah_ocr: img = cv2.imread(dir_img) - if self.dir_out_image_text: - out_image_with_text = os.path.join(self.dir_out_image_text, file_name+'.png') + if dir_out_image_text: + out_image_with_text = os.path.join(dir_out_image_text, file_name+'.png') image_text = Image.new("RGB", (img.shape[1], img.shape[0]), "white") draw = ImageDraw.Draw(image_text) total_bb_coordinates = [] @@ -5458,7 +5429,7 @@ class Eynollah_ocr: textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) x,y,w,h = cv2.boundingRect(textline_coords) - if self.dir_out_image_text: + if dir_out_image_text: total_bb_coordinates.append([x,y,w,h]) h2w_ratio = h/float(w) @@ -5580,7 +5551,7 @@ class Eynollah_ocr: unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - if self.dir_out_image_text: + if dir_out_image_text: font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = ImageFont.truetype(font_path, 40) @@ -5708,18 +5679,10 @@ class Eynollah_ocr: img_size=(image_width, image_height) - for ind_img in ls_imgs: - if self.dir_in: - file_name = Path(ind_img).stem - dir_img = os.path.join(self.dir_in, ind_img) - else: - file_name = Path(self.image_filename).stem - dir_img = self.image_filename - - #file_name = Path(ind_img).stem - #dir_img = os.path.join(self.dir_in, ind_img) - dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') - out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') + for dir_img in ls_imgs: + file_name = Path(dir_img).stem + dir_xml = os.path.join(dir_xmls, file_name+'.xml') + out_file_ocr = os.path.join(dir_out, file_name+'.xml') if os.path.exists(out_file_ocr): if overwrite: @@ -5729,13 +5692,13 @@ class Eynollah_ocr: continue img = cv2.imread(dir_img) - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: cropped_lines_bin = [] - dir_img_bin = os.path.join(self.dir_in_bin, file_name+'.png') + dir_img_bin = os.path.join(dir_in_bin, file_name+'.png') img_bin = cv2.imread(dir_img_bin) - if self.dir_out_image_text: - out_image_with_text = os.path.join(self.dir_out_image_text, file_name+'.png') + if dir_out_image_text: + out_image_with_text = os.path.join(dir_out_image_text, file_name+'.png') image_text = Image.new("RGB", (img.shape[1], img.shape[0]), "white") draw = ImageDraw.Draw(image_text) total_bb_coordinates = [] @@ -5779,13 +5742,13 @@ class Eynollah_ocr: if type_textregion=='drop-capital': angle_degrees = 0 - if self.dir_out_image_text: + if dir_out_image_text: total_bb_coordinates.append([x,y,w,h]) w_scaled = w * image_height/float(h) img_poly_on_img = np.copy(img) - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_poly_on_img_bin = np.copy(img_bin) img_crop_bin = img_poly_on_img_bin[y:y+h, x:x+w, :] @@ -5808,7 +5771,7 @@ class Eynollah_ocr: img_crop = rotate_image_with_padding(img_crop, better_des_slope ) - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) @@ -5823,13 +5786,13 @@ class Eynollah_ocr: if not self.do_not_mask_with_textline_contour: img_crop[mask_poly==0] = 255 - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] if not self.do_not_mask_with_textline_contour: img_crop_bin[mask_poly==0] = 255 if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) @@ -5839,14 +5802,14 @@ class Eynollah_ocr: better_des_slope = 0 if not self.do_not_mask_with_textline_contour: img_crop[mask_poly==0] = 255 - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: if not self.do_not_mask_with_textline_contour: img_crop_bin[mask_poly==0] = 255 if type_textregion=='drop-capital': pass else: if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) else: img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) @@ -5861,14 +5824,12 @@ class Eynollah_ocr: cropped_lines_ver_index.append(0) cropped_lines_meging_indexing.append(0) - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) else: - if self.prediction_with_both_of_rgb_and_bin: - splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, img_crop_bin, prediction_with_both_of_rgb_and_bin=self.prediction_with_both_of_rgb_and_bin) - else: - splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) + splited_images, splited_images_bin = return_textlines_split_if_needed( + img_crop, img_crop_bin if dir_in_bin is not None else None) if splited_images: img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) cropped_lines.append(img_fin) @@ -5889,7 +5850,7 @@ class Eynollah_ocr: else: cropped_lines_ver_index.append(0) - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[0], image_height, image_width) cropped_lines_bin.append(img_fin) img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[1], image_height, image_width) @@ -5905,7 +5866,7 @@ class Eynollah_ocr: else: cropped_lines_ver_index.append(0) - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) @@ -5918,29 +5879,15 @@ class Eynollah_ocr: if cheild_text.tag.endswith("Unicode"): textline_text = cheild_text.text if textline_text: - if self.do_not_mask_with_textline_contour: - if self.pref_of_dataset: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.txt'), 'w') as text_file: - text_file.write(textline_text) - - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'.png'), img_crop ) - else: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.txt'), 'w') as text_file: - text_file.write(textline_text) - - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.png'), img_crop ) - else: - if self.pref_of_dataset: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.txt'), 'w') as text_file: - text_file.write(textline_text) - - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_'+self.pref_of_dataset+'_masked.png'), img_crop ) - else: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.txt'), 'w') as text_file: - text_file.write(textline_text) - - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'_masked.png'), img_crop ) + base_name = os.path.join(dir_out, file_name + '_line_' + str(indexer_textlines)) + if self.pref_of_dataset: + base_name += '_' + self.pref_of_dataset + if not self.do_not_mask_with_textline_contour: + base_name += '_masked' + with open(base_name + '.txt', 'w') as text_file: + text_file.write(textline_text) + cv2.imwrite(base_name + '.png', img_crop) indexer_textlines+=1 if not self.export_textline_images_and_text: @@ -5971,7 +5918,7 @@ class Eynollah_ocr: else: imgs_ver_flipped = None - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: imgs_bin = cropped_lines_bin[n_start:] imgs_bin = np.array(imgs_bin) imgs_bin = imgs_bin.reshape(imgs_bin.shape[0], image_height, image_width, 3) @@ -6001,7 +5948,7 @@ class Eynollah_ocr: imgs_ver_flipped = None - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: imgs_bin = cropped_lines_bin[n_start:n_end] imgs_bin = np.array(imgs_bin).reshape(self.b_s, image_height, image_width, 3) @@ -6040,7 +5987,7 @@ class Eynollah_ocr: if len(indices_where_flipped_conf_value_is_higher)>0: indices_to_be_replaced = indices_ver[indices_where_flipped_conf_value_is_higher] preds[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) if len(indices_ver)>0: @@ -6087,7 +6034,7 @@ class Eynollah_ocr: extracted_texts.append("") extracted_conf_value.append(0) del cropped_lines - if self.prediction_with_both_of_rgb_and_bin: + if dir_in_bin is not None: del cropped_lines_bin gc.collect() @@ -6100,7 +6047,7 @@ class Eynollah_ocr: unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - if self.dir_out_image_text: + if dir_out_image_text: font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = ImageFont.truetype(font_path, 40) diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index d974650..4fa99f7 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -109,13 +109,13 @@ def fit_text_single_line(draw, text, font_path, max_width, max_height): return ImageFont.truetype(font_path, 10) # Smallest font fallback -def return_textlines_split_if_needed(textline_image, textline_image_bin, prediction_with_both_of_rgb_and_bin=False): +def return_textlines_split_if_needed(textline_image, textline_image_bin=None): split_point = return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image) if split_point: image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) - if prediction_with_both_of_rgb_and_bin: + if textline_image_bin is not None: image1_bin = textline_image_bin[:, :split_point,:]# image.crop((0, 0, width2, height)) image2_bin = textline_image_bin[:, split_point:,:]#image.crop((width1, 0, width, height)) return [image1, image2], [image1_bin, image2_bin] From 1dcc7b5795d92619cd87699e6030cea088441f3c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 16:38:43 +0200 Subject: [PATCH 223/374] ocr CLI: make --model vs --model_name xor --- src/eynollah/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index a0608f9..3436250 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -495,7 +495,6 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ "--model_name", help="Specific model file path to use for OCR", type=click.Path(exists=True, file_okay=False), - required=True, ) @click.option( "--tr_ocr", @@ -540,7 +539,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ def ocr(image, dir_in, dir_in_bin, dir_xmls, out, dir_out_image_text, overwrite, model, model_name, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, batch_size, dataset_abbrevation, min_conf_value_of_textline_text, log_level): initLogging() - assert not model or not model_name, "model directory -m can not be set alongside specific model name --model_name" + assert bool(model) != bool(model_name), "Either -m (model directory) or --model_name (specific model name) must be provided." assert not export_textline_images_and_text or not tr_ocr, "Exporting textline and text -etit can not be set alongside transformer ocr -tr_ocr" assert not export_textline_images_and_text or not model, "Exporting textline and text -etit can not be set alongside model -m" assert not export_textline_images_and_text or not batch_size, "Exporting textline and text -etit can not be set alongside batch size -bs" From 2d14d57e4f42988e19cbc976e8b5174dec671b1b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 19:52:50 +0200 Subject: [PATCH 224/374] ocr: minimal debug logging --- src/eynollah/eynollah.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 533b38f..6191b8e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5442,7 +5442,7 @@ class Eynollah_ocr: img_crop = img_poly_on_img[y:y+h, x:x+w, :] img_crop[mask_poly==0] = 255 - + self.logger.debug("processing %d lines for '%s'", len(cropped_lines), nn.attrib['id']) if h2w_ratio > 0.1: cropped_lines.append(resize_image(img_crop, tr_ocr_input_height_and_width, tr_ocr_input_height_and_width) ) cropped_lines_meging_indexing.append(0) @@ -5961,6 +5961,7 @@ class Eynollah_ocr: imgs_bin_ver_flipped = None + self.logger.debug("processing next %d lines", len(imgs)) preds = self.prediction_model.predict(imgs, verbose=0) if len(indices_ver)>0: From 5c7e1f21fb5c36c4012eb8b7231af47166da2820 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 19:53:19 +0200 Subject: [PATCH 225/374] test_run: add tests for ocr --- tests/test_run.py | 80 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 370deef..cd24225 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -7,6 +7,7 @@ from eynollah.cli import ( binarization as binarization_cli, enhancement as enhancement_cli, machine_based_reading_order as mbreorder_cli, + ocr as ocr_cli, ) from click.testing import CliRunner from ocrd_modelfactory import page_from_file @@ -76,7 +77,7 @@ def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): return logrec.name == 'eynollah' runner = CliRunner() with caplog.filtering(only_eynollah): - result = runner.invoke(layout_cli, args) + result = runner.invoke(layout_cli, args, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Job done in')]) == 2 @@ -104,7 +105,7 @@ def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, ca with subtests.test(#msg="test CLI", options=options): with caplog.filtering(only_eynollah): - result = runner.invoke(binarization_cli, args + options) + result = runner.invoke(binarization_cli, args + options, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert any(True for logmsg in logmsgs if logmsg.startswith('Predicting')) @@ -130,7 +131,7 @@ def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, c return logrec.name == 'SbbBinarizer' runner = CliRunner() with caplog.filtering(only_eynollah): - result = runner.invoke(binarization_cli, args) + result = runner.invoke(binarization_cli, args, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Predicting')]) == 2 @@ -159,7 +160,7 @@ def test_run_eynollah_enhancement_filename(tmp_path, subtests, pytestconfig, cap with subtests.test(#msg="test CLI", options=options): with caplog.filtering(only_eynollah): - result = runner.invoke(enhancement_cli, args + options) + result = runner.invoke(enhancement_cli, args + options, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert any(True for logmsg in logmsgs if logmsg.startswith('Image was enhanced')), logmsgs @@ -185,7 +186,7 @@ def test_run_eynollah_enhancement_directory(tmp_path, subtests, pytestconfig, ca return logrec.name == 'enhancement' runner = CliRunner() with caplog.filtering(only_eynollah): - result = runner.invoke(enhancement_cli, args) + result = runner.invoke(enhancement_cli, args, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Image was enhanced')]) == 2 @@ -206,7 +207,7 @@ def test_run_eynollah_mbreorder_filename(tmp_path, subtests, pytestconfig, caplo return logrec.name == 'mbreorder' runner = CliRunner() with caplog.filtering(only_eynollah): - result = runner.invoke(mbreorder_cli, args) + result = runner.invoke(mbreorder_cli, args, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] # FIXME: mbreorder has no logging! @@ -235,9 +236,74 @@ def test_run_eynollah_mbreorder_directory(tmp_path, subtests, pytestconfig, capl return logrec.name == 'mbreorder' runner = CliRunner() with caplog.filtering(only_eynollah): - result = runner.invoke(mbreorder_cli, args) + result = runner.invoke(mbreorder_cli, args, catch_exceptions=False) assert result.exit_code == 0, result.stdout logmsgs = [logrec.message for logrec in caplog.records] # FIXME: mbreorder has no logging! #assert len([logmsg for logmsg in logmsgs if logmsg.startswith('???')]) == 2 assert len(list(outdir.iterdir())) == 2 + +def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): + infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') + outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.xml') + outrenderfile = tmp_path.joinpath('render').joinpath('kant_aufklaerung_1784_0020.xml') + outrenderfile.parent.mkdir() + args = [ + '-m', EYNOLLAH_MODELS, + '-i', str(infile), + '-dx', str(infile.parent), + '-o', str(outfile.parent), + # subtests write to same location + '--overwrite', + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.DEBUG) + def only_eynollah(logrec): + return logrec.name == 'eynollah' + runner = CliRunner() + for options in [ + [], # defaults + ["-doit", str(outrenderfile.parent)], + ["-trocr"], + ]: + with subtests.test(#msg="test CLI", + options=options): + with caplog.filtering(only_eynollah): + result = runner.invoke(ocr_cli, args + options, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + # FIXME: ocr has no logging! + #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs + assert outfile.exists() + if "-doit" in options: + assert outrenderfile.exists() + #in_tree = page_from_file(str(infile)).etree + #in_order = in_tree.xpath("//page:OrderedGroup//@regionRef", namespaces=NS) + out_tree = page_from_file(str(outfile)).etree + out_texts = out_tree.xpath("//page:TextLine/page:TextEquiv[last()]/page:Unicode/text()", namespaces=NS) + assert len(out_texts) >= 2, ("result is inaccurate", out_texts) + assert sum(map(len, out_texts)) > 100, ("result is inaccurate", out_texts) + +def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): + indir = testdir.joinpath('resources') + outdir = tmp_path + args = [ + '-m', EYNOLLAH_MODELS, + '-di', str(indir), + '-dx', str(indir), + '-o', str(outdir), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'eynollah' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(ocr_cli, args, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + # FIXME: ocr has no logging! + #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs + assert len(list(outdir.iterdir())) == 2 From 11de8a025d8e7dc5a3be54cd8009144f947c7a24 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 20:11:48 +0200 Subject: [PATCH 226/374] Adapt ocrd-eynollah-segment for release --- src/eynollah/ocrd-tool.json | 12 +++++++++++- src/eynollah/processor.py | 9 +++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index af5e03f..e5077e9 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -82,13 +82,23 @@ } }, "resources": [ + { + "url": "https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1", + "name": "eynollah_layout_v0_5_0", + "type": "archive", + "path_in_archive": "eynollah_layout_v0_5_0", + "size": 3525684179, + "description": "Models for layout detection, reading order detection, textline detection, page extraction, column classification, table detection, binarization, image enhancement", + "version_range": ">= v0.5.0" + }, { "description": "models for eynollah (TensorFlow SavedModel format)", "url": "https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz", "name": "default", "size": 1894627041, "type": "archive", - "path_in_archive": "models_eynollah" + "path_in_archive": "models_eynollah", + "version_range": ">= v0.3.0, < v0.5.0" } ] }, diff --git a/src/eynollah/processor.py b/src/eynollah/processor.py index c2922c1..12c7356 100644 --- a/src/eynollah/processor.py +++ b/src/eynollah/processor.py @@ -1,6 +1,7 @@ +from functools import cached_property from typing import Optional from ocrd_models import OcrdPage -from ocrd import Processor, OcrdPageResult +from ocrd import OcrdPageResultImage, Processor, OcrdPageResult from .eynollah import Eynollah, EynollahXmlWriter @@ -9,8 +10,8 @@ class EynollahProcessor(Processor): # already employs GPU (without singleton process atm) max_workers = 1 - @property - def executable(self): + @cached_property + def executable(self) -> str: return 'ocrd-eynollah-segment' def setup(self) -> None: @@ -20,7 +21,6 @@ class EynollahProcessor(Processor): "and parameter 'light_version' (faster+simpler method for main region detection and deskewing)") self.eynollah = Eynollah( self.resolve_resource(self.parameter['models']), - logger=self.logger, allow_enhancement=self.parameter['allow_enhancement'], curved_line=self.parameter['curved_line'], right2left=self.parameter['right_to_left'], @@ -33,6 +33,7 @@ class EynollahProcessor(Processor): headers_off=self.parameter['headers_off'], tables=self.parameter['tables'], ) + self.eynollah.logger = self.logger self.eynollah.plotter = None def shutdown(self): From e6ee26fde35d93584f295d47f3d8f85c1d65124a Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 20:35:54 +0200 Subject: [PATCH 227/374] make models: adapt to zenodo/v0.5.0 --- Makefile | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index f78d7d1..6566293 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,10 @@ DOCKER ?= docker #SEG_MODEL := https://qurator-data.de/eynollah/2021-04-25/models_eynollah.tar.gz #SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah_renamed.tar.gz -SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah.tar.gz +# SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah.tar.gz #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.0/models_eynollah.tar.gz #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz +SEG_MODEL := https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1 BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip @@ -28,7 +29,7 @@ help: @echo " install Install package with pip" @echo " install-dev Install editable with pip" @echo " deps-test Install test dependencies with pip" - @echo " models Download and extract models to $(CURDIR)/models_eynollah" + @echo " models Download and extract models to $(CURDIR)/models_layout_v0_5_0" @echo " smoke-test Run simple CLI check" @echo " ocrd-test Run OCR-D CLI check" @echo " test Run unit tests" @@ -44,13 +45,13 @@ help: # END-EVAL -# Download and extract models to $(PWD)/models_eynollah -models: models_eynollah default-2021-03-09 +# Download and extract models to $(PWD)/models_layout_v0_5_0 +models: models_layout_v0_5_0 default-2021-03-09 -models_eynollah: models_eynollah.tar.gz - tar zxf models_eynollah.tar.gz +models_layout_v0_5_0: models_layout_v0_5_0.tar.gz + tar zxf models_layout_v0_5_0.tar.gz -models_eynollah.tar.gz: +models_layout_v0_5_0.tar.gz: wget $(SEG_MODEL) default-2021-03-09: $(notdir $(BIN_MODEL)) @@ -73,20 +74,20 @@ install: install-dev: $(PIP) install -e .$(and $(EXTRAS),[$(EXTRAS)]) -deps-test: models_eynollah +deps-test: models_layout_v0_5_0 $(PIP) install -r requirements-test.txt smoke-test: TMPDIR != mktemp -d smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif # layout analysis: - eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/models_eynollah + eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/models_layout_v0_5_0 fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$(basename $( Date: Thu, 25 Sep 2025 20:51:02 +0200 Subject: [PATCH 228/374] enhancement/mbreorder: make all path options kwargs to run() instead of attributes --- src/eynollah/cli.py | 20 +++++++-------- src/eynollah/image_enhancer.py | 23 +++++++++-------- src/eynollah/mb_ro_on_layout.py | 45 +++++++++++++++------------------ 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 3436250..93bb676 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -47,14 +47,14 @@ def main(): def machine_based_reading_order(input, dir_in, out, model, log_level): assert bool(input) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." - orderer = machine_based_reading_order_on_layout(model, dir_out=out) + orderer = machine_based_reading_order_on_layout(model) if log_level: orderer.logger.setLevel(getLevelName(log_level)) - if dir_in: - orderer.run(dir_in=dir_in) - else: - orderer.run(xml_filename=input) + orderer.run(xml_filename=input, + dir_in=dir_in, + dir_out=out, + ) @main.command() @@ -156,17 +156,17 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low initLogging() enhancer = Enhancer( model, - dir_out=out, num_col_upper=num_col_upper, num_col_lower=num_col_lower, save_org_scale=save_org_scale, ) if log_level: enhancer.logger.setLevel(getLevelName(log_level)) - if dir_in: - enhancer.run(dir_in=dir_in, overwrite=overwrite) - else: - enhancer.run(image_filename=image, overwrite=overwrite) + enhancer.run(overwrite=overwrite, + dir_in=dir_in, + image_filename=image, + dir_out=out, + ) @main.command() @click.option( diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index 5a06d59..89dde16 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -11,7 +11,6 @@ from functools import partial from pathlib import Path from multiprocessing import cpu_count import gc -from loky import ProcessPoolExecutor import cv2 import numpy as np from ocrd_utils import getLogger, tf_disable_interactive_logs @@ -33,13 +32,11 @@ class Enhancer: def __init__( self, dir_models : str, - dir_out : Optional[str] = None, num_col_upper : Optional[int] = None, num_col_lower : Optional[int] = None, save_org_scale : bool = False, logger : Optional[Logger] = None, ): - self.dir_out = dir_out self.input_binary = False self.light_version = False self.save_org_scale = save_org_scale @@ -53,9 +50,6 @@ class Enhancer: self.num_col_lower = num_col_lower self.logger = logger if logger else getLogger('enhancement') - # for parallelization of CPU-intensive tasks: - self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) - atexit.register(self.executor.shutdown) self.dir_models = dir_models self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" @@ -94,9 +88,9 @@ class Enhancer: if dpi is not None: self.dpi = dpi - def reset_file_name_dir(self, image_filename): + def reset_file_name_dir(self, image_filename, dir_out): self.cache_images(image_filename=image_filename) - self.output_filename = os.path.join(self.dir_out, Path(image_filename).stem +'.png') + self.output_filename = os.path.join(dir_out, Path(image_filename).stem +'.png') def imread(self, grayscale=False, uint8=True): key = 'img' @@ -694,7 +688,12 @@ class Enhancer: return img_res - def run(self, image_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): + def run(self, + overwrite: bool = False, + image_filename: Optional[str] = None, + dir_in: Optional[str] = None, + dir_out: Optional[str] = None, + ): """ Get image and scales, then extract the page of scanned image """ @@ -702,7 +701,9 @@ class Enhancer: t0_tot = time.time() if dir_in: - ls_imgs = list(filter(is_image_filename, os.listdir(dir_in))) + ls_imgs = [os.path.join(dir_in, image_filename) + for image_filename in filter(is_image_filename, + os.listdir(dir_in))] elif image_filename: ls_imgs = [image_filename] else: @@ -712,7 +713,7 @@ class Enhancer: self.logger.info(img_filename) t0 = time.time() - self.reset_file_name_dir(os.path.join(dir_in or "", img_filename)) + self.reset_file_name_dir(img_filename, dir_out) #print("text region early -11 in %.1fs", time.time() - t0) if os.path.exists(self.output_filename): diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index 6d72614..45db8e4 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -10,7 +10,6 @@ import atexit from functools import partial from pathlib import Path from multiprocessing import cpu_count -from loky import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 import numpy as np @@ -35,15 +34,9 @@ class machine_based_reading_order_on_layout: def __init__( self, dir_models : str, - dir_out : Optional[str] = None, logger : Optional[Logger] = None, ): - self.dir_out = dir_out - self.logger = logger if logger else getLogger('mbreorder') - # for parallelization of CPU-intensive tasks: - self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) - atexit.register(self.executor.shutdown) self.dir_models = dir_models self.model_reading_order_dir = dir_models + "/model_eynollah_reading_order_20250824" @@ -56,9 +49,6 @@ class machine_based_reading_order_on_layout: self.model_reading_order = self.our_load_model(self.model_reading_order_dir) self.light_version = True - - - @staticmethod def our_load_model(model_file): if model_file.endswith('.h5') and Path(model_file[:-3]).exists(): @@ -70,10 +60,8 @@ class machine_based_reading_order_on_layout: model = load_model(model_file, compile=False, custom_objects={ "PatchEncoder": PatchEncoder, "Patches": Patches}) return model - - + def read_xml(self, xml_file): - file_name = Path(xml_file).stem tree1 = ET.parse(xml_file, parser = ET.XMLParser(encoding='utf-8')) root1=tree1.getroot() alltags=[elem.tag for elem in root1.iter()] @@ -495,7 +483,7 @@ class machine_based_reading_order_on_layout: img_poly=cv2.fillPoly(img, pts =co_img, color=(4,4,4)) img_poly=cv2.fillPoly(img, pts =co_sep, color=(5,5,5)) - return tree1, root1, bb_coord_printspace, file_name, id_paragraph, id_header+id_heading, co_text_paragraph, co_text_header+co_text_heading,\ + return tree1, root1, bb_coord_printspace, id_paragraph, id_header+id_heading, co_text_paragraph, co_text_header+co_text_heading,\ tot_region_ref,x_len, y_len,index_tot_regions, img_poly def return_indexes_of_contours_loctaed_inside_another_list_of_contours(self, contours, contours_loc, cx_main_loc, cy_main_loc, indexes_loc): @@ -744,7 +732,12 @@ class machine_based_reading_order_on_layout: - def run(self, xml_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): + def run(self, + overwrite: bool = False, + xml_filename: Optional[str] = None, + dir_in: Optional[str] = None, + dir_out: Optional[str] = None, + ): """ Get image and scales, then extract the page of scanned image """ @@ -752,7 +745,9 @@ class machine_based_reading_order_on_layout: t0_tot = time.time() if dir_in: - ls_xmls = list(filter(is_xml_filename, os.listdir(dir_in))) + ls_xmls = [os.path.join(dir_in, xml_filename) + for xml_filename in filter(is_xml_filename, + os.listdir(dir_in))] elif xml_filename: ls_xmls = [xml_filename] else: @@ -761,13 +756,11 @@ class machine_based_reading_order_on_layout: for xml_filename in ls_xmls: self.logger.info(xml_filename) t0 = time.time() - - if dir_in: - xml_file = os.path.join(dir_in, xml_filename) - else: - xml_file = xml_filename - - tree_xml, root_xml, bb_coord_printspace, file_name, id_paragraph, id_header, co_text_paragraph, co_text_header, tot_region_ref, x_len, y_len, index_tot_regions, img_poly = self.read_xml(xml_file) + + file_name = Path(xml_filename).stem + (tree_xml, root_xml, bb_coord_printspace, id_paragraph, id_header, + co_text_paragraph, co_text_header, tot_region_ref, + x_len, y_len, index_tot_regions, img_poly) = self.read_xml(xml_filename) id_all_text = id_paragraph + id_header @@ -810,7 +803,11 @@ class machine_based_reading_order_on_layout: alltags=[elem.tag for elem in root_xml.iter()] ET.register_namespace("",name_space) - tree_xml.write(os.path.join(self.dir_out, file_name+'.xml'),xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) + tree_xml.write(os.path.join(dir_out, file_name+'.xml'), + xml_declaration=True, + method='xml', + encoding="utf8", + default_namespace=None) #sys.exit() From 9303ded11f98e0a04eb7f09b424d62812fb84d66 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 21:12:52 +0200 Subject: [PATCH 229/374] ocrd-tool.json: use models_layout instead of eynollah_layouts for consistency --- Makefile | 2 +- src/eynollah/ocrd-tool.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6566293..20f4755 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ models_layout_v0_5_0: models_layout_v0_5_0.tar.gz tar zxf models_layout_v0_5_0.tar.gz models_layout_v0_5_0.tar.gz: - wget $(SEG_MODEL) + wget -O $@ $(SEG_MODEL) default-2021-03-09: $(notdir $(BIN_MODEL)) unzip $(notdir $(BIN_MODEL)) diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index e5077e9..fbc6c1a 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -84,9 +84,9 @@ "resources": [ { "url": "https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1", - "name": "eynollah_layout_v0_5_0", + "name": "models_layout_v0_5_0", "type": "archive", - "path_in_archive": "eynollah_layout_v0_5_0", + "path_in_archive": "models_layout_v0_5_0", "size": 3525684179, "description": "Models for layout detection, reading order detection, textline detection, page extraction, column classification, table detection, binarization, image enhancement", "version_range": ">= v0.5.0" From 5c0ab509c4f3620f23d69655f339fae3cd2f02a4 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 21:17:32 +0200 Subject: [PATCH 230/374] CI: Update model name --- .github/workflows/test-eynollah.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index b27586c..b270ab1 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/cache@v4 id: seg_model_cache with: - path: models_eynollah + path: models_layout_v0_5_0 key: ${{ runner.os }}-models - uses: actions/cache@v4 id: bin_model_cache From 0bb1fb1a053a464675c9c41ea21833763cd37b01 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 21:47:15 +0200 Subject: [PATCH 231/374] tests: adapt to layout/ocr model split --- Makefile | 5 +++-- tests/test_run.py | 25 +++++++++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index 20f4755..41f46d6 100644 --- a/Makefile +++ b/Makefile @@ -115,8 +115,9 @@ ocrd-test: tests/resources/kant_aufklaerung_1784_0020.tif $(RM) -r $(TMPDIR) # Run unit tests -test: export EYNOLLAH_MODELS=$(CURDIR)/models_layout_v0_5_0 -test: export SBBBIN_MODELS=$(CURDIR)/default-2021-03-09 +test: export MODELS_LAYOUT=$(CURDIR)/models_layout_v0_5_0 +test: export MODELS_OCR=$(CURDIR)/models_ocr_v0_5_0 +test: export MODELS_BIN=$(CURDIR)/default-2021-03-09 test: $(PYTHON) -m pytest tests --durations=0 --continue-on-collection-errors $(PYTEST_ARGS) diff --git a/tests/test_run.py b/tests/test_run.py index cd24225..aea5808 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -15,14 +15,15 @@ from ocrd_models.constants import NAMESPACES as NS testdir = Path(__file__).parent.resolve() -EYNOLLAH_MODELS = environ.get('EYNOLLAH_MODELS', str(testdir.joinpath('..', 'models_eynollah').resolve())) -SBBBIN_MODELS = environ.get('SBBBIN_MODELS', str(testdir.joinpath('..', 'default-2021-03-09').resolve())) +MODELS_LAYOUT = environ.get('MODELS_LAYOUT', str(testdir.joinpath('..', 'models_layout_v0_5_0').resolve())) +MODELS_OCR = environ.get('MODELS_OCR', str(testdir.joinpath('..', 'models_ocr_v0_5_0').resolve())) +MODELS_BIN = environ.get('MODELS_BIN', str(testdir.joinpath('..', 'default-2021-03-09').resolve())) def test_run_eynollah_layout_filename(tmp_path, subtests, pytestconfig, caplog): infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path / 'kant_aufklaerung_1784_0020.xml' args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_LAYOUT, '-i', str(infile), '-o', str(outfile.parent), # subtests write to same location @@ -66,7 +67,7 @@ def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_LAYOUT, '-di', str(indir), '-o', str(outdir), ] @@ -88,7 +89,7 @@ def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, ca infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') args = [ - '-m', SBBBIN_MODELS, + '-m', MODELS_BIN, '-i', str(infile), '-o', str(outfile), ] @@ -120,7 +121,7 @@ def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, c indir = testdir.joinpath('resources') outdir = tmp_path args = [ - '-m', SBBBIN_MODELS, + '-m', MODELS_BIN, '-di', str(indir), '-o', str(outdir), ] @@ -141,7 +142,7 @@ def test_run_eynollah_enhancement_filename(tmp_path, subtests, pytestconfig, cap infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_LAYOUT, '-i', str(infile), '-o', str(outfile.parent), # subtests write to same location @@ -175,7 +176,7 @@ def test_run_eynollah_enhancement_directory(tmp_path, subtests, pytestconfig, ca indir = testdir.joinpath('resources') outdir = tmp_path args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_LAYOUT, '-di', str(indir), '-o', str(outdir), ] @@ -196,7 +197,7 @@ def test_run_eynollah_mbreorder_filename(tmp_path, subtests, pytestconfig, caplo infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.xml') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.xml') args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_LAYOUT, '-i', str(infile), '-o', str(outfile.parent), ] @@ -225,7 +226,7 @@ def test_run_eynollah_mbreorder_directory(tmp_path, subtests, pytestconfig, capl indir = testdir.joinpath('resources') outdir = tmp_path args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_LAYOUT, '-di', str(indir), '-o', str(outdir), ] @@ -249,7 +250,7 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): outrenderfile = tmp_path.joinpath('render').joinpath('kant_aufklaerung_1784_0020.xml') outrenderfile.parent.mkdir() args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_OCR, '-i', str(infile), '-dx', str(infile.parent), '-o', str(outfile.parent), @@ -289,7 +290,7 @@ def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path args = [ - '-m', EYNOLLAH_MODELS, + '-m', MODELS_OCR, '-di', str(indir), '-dx', str(indir), '-o', str(outdir), From b4d460ca79c8b7361e2ff34477f33fcb62a7a319 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 22:16:38 +0200 Subject: [PATCH 232/374] makefile forgot the OCR models --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 41f46d6..a920615 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ SEG_MODEL := https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar. BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip +OCR_MODEL := https://zenodo.org/records/17194824/files/models_ocr_v0_5_0.tar.gz?download=1 + PYTEST_ARGS ?= -vv # BEGIN-EVAL makefile-parser --make-help Makefile @@ -46,7 +48,7 @@ help: # Download and extract models to $(PWD)/models_layout_v0_5_0 -models: models_layout_v0_5_0 default-2021-03-09 +models: models_layout_v0_5_0 models_ocr_v0_5_0 default-2021-03-09 models_layout_v0_5_0: models_layout_v0_5_0.tar.gz tar zxf models_layout_v0_5_0.tar.gz @@ -54,6 +56,12 @@ models_layout_v0_5_0: models_layout_v0_5_0.tar.gz models_layout_v0_5_0.tar.gz: wget -O $@ $(SEG_MODEL) +models_ocr_v0_5_0: models_ocr_v0_5_0.tar.gz + tar zxf models_ocr_v0_5_0.tar.gz + +models_ocr_v0_5_0.tar.gz: + wget -O $@ $(OCR_MODEL) + default-2021-03-09: $(notdir $(BIN_MODEL)) unzip $(notdir $(BIN_MODEL)) mkdir $@ From 4c6405713a087781f6bdabf6a31d00d81f7b9a0b Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 25 Sep 2025 22:19:36 +0200 Subject: [PATCH 233/374] ci: ocr models --- .github/workflows/test-eynollah.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index b270ab1..042e508 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -29,13 +29,18 @@ jobs: with: path: models_layout_v0_5_0 key: ${{ runner.os }}-models + - uses: actions/cache@v4 + id: ocr_model_cache + with: + path: models_ocr_v0_5_0 + key: ${{ runner.os }}-models - uses: actions/cache@v4 id: bin_model_cache with: path: default-2021-03-09 key: ${{ runner.os }}-modelbin - name: Download models - if: steps.seg_model_cache.outputs.cache-hit != 'true' || steps.bin_model_cache.outputs.cache-hit != 'true' + if: steps.seg_model_cache.outputs.cache-hit != 'true' || steps.bin_model_cache.outputs.cache-hit != 'true' || steps.ocr_model_cache.outputs.cache-hit != true run: make models - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 From 480daa4c7c92e22c16f1b7fc56cca48177953a3d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 25 Sep 2025 22:25:05 +0200 Subject: [PATCH 234/374] test_run: make ocr -doit work (add truetype file) --- pyproject.toml | 2 +- src/eynollah/Charis-Regular.ttf | Bin 0 -> 878076 bytes src/eynollah/eynollah.py | 23 +++++++++++++++++------ tests/test_run.py | 2 +- 4 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 src/eynollah/Charis-Regular.ttf diff --git a/pyproject.toml b/pyproject.toml index 4da39ef..8a63543 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ optional-dependencies.test = {file = ["requirements-test.txt"]} where = ["src"] [tool.setuptools.package-data] -"*" = ["*.json", '*.yml', '*.xml', '*.xsd'] +"*" = ["*.json", '*.yml', '*.xml', '*.xsd', '*.ttf'] [tool.coverage.run] branch = true diff --git a/src/eynollah/Charis-Regular.ttf b/src/eynollah/Charis-Regular.ttf new file mode 100644 index 0000000000000000000000000000000000000000..a4e75a450996c466d8eaa8528943c77590820bac GIT binary patch literal 878076 zcmdpf3!IJB{`a~(w|$vCV+_XJ4Na5eI1)qBnEULR*|X=q_a@DK5JGb#hK3|bk|arz zBuSEnG)a<>B!nZ$aU92yBsr3#nfJTa-ouQePMy>H{@?fAzt8%u-}*i4SZmDI_X2DLLgu%L|_pLc0*6O-XfT=e2!-UPp+&ijdEzq~;YQ>^`_|Ga-NL zLCB)Y?7Tm8FIhip5TOq&KqYi)$CSVG9u(`93- zqmBoLmAvUUp-B)RPtHa-rK0r+C z4i2s;DeqIfc`32{ScmOB2SX9s>V`U`Pa)lD@bK!f+NXaWLM)d82)V6(*xh9%*~WrO zV(q;H=^evM#*Pdu4Y-q7@12MIQ6ox*SDYFh^d-{k2#I%$y!)Q&@v3Jlv92=+c~u!X zx?*HlN#QHRufmG_-w;X|u|XodPK|>648$*?JxRVIoSdQ!gwjUZNH`;`KcOsvB|uJO zk3*i#Wtxzk3T%;92c4@;P57&l69-$qA z{Dt-fB;8}u6>ch>KRT&veYo}{ma{D%Gpkes z4DvyfAE73j$)9jjfGH4iYf}*9Hm3U^k28&fJl<3Td4g#INgorp=H)Fl~jr&9oizN2a}y_nGh%O=nC$K)z_Y2>D0TkB~2!EL{mrc*2bqH)w=uVY+}7L{ayv8bzd6(#3OURi4mrXc0Xfnf1-XOykC3~Y zZ-(5%d<*2B<~Yc`&9_62H=_=iQ_NYA9p)U!x#nERd1llHbAh=CvdcUW@*QSanE6g~ zG2{|+DdaM98RT+v739I@VUUNLheIA=9s&7o^GL{}%+-+pWd0N6G3GIl?={~Gd93+9 z$m7i8AdfeXhg@SueKX&0Mtw8an(9t9(HG1vPYk^+d^IDzP-n^FPwK1=C zd2PvSMPB>yT8`Idyw>8i2d^b~ZNOXOz4hB$v%U4$TZ2V?U4fdn61CP_U#V#aYAQ8- zi#qD9k)kd#+;IgpQq)C%@K)wl;G!;4b9*z|0$Jy%`DWBMYK}uaqvrmoVWMtjgXf@z zQFA`(7B#!gZtwx-0pNFVg0c5#ke!eOfQ?3n_e-k!2Nm6w953lX*Fzmi)kzF(Z{BprcX?} zOuJ2cOrM(e;(nbq{nK>L^quLv>3h=!(+|R$g%#tDwZffhFYXZTPaN(`f;rKgWKI_M z26rYOcVqzWL>cbDP+a*aT=7`*Gq}pTEn`rxviu$X+5S#{afR_(0b0V1Xwhb(WqQ%{ zD%zDZrXS7R9AFMKw?>N)Yz`4^0qnk~xfiV535#}_J?4StV)GF5U9i#7=4x2veXzd! zVQI6>kDF)1vgVqfGS4$VZJuvlU|t9-g6#;Kfvt?R+$*dBwLKU0TJ+au5<$8Xo|KZA zPn>1_0h?pZ#f_{*wc%pvj_xwo!9Y|PKMp6_Vn3tOX zPBRHhcRJH(W?_1Mlt^I@s-Ki+Nm;Zvrl^ggv}Umae}hsdxQO$8vC9{u)f43e&XxNy zobNWDWTS0~1`k2(0~bX~=;h>APP@_VzGJ8~ljh_6%ISW&O_lO6T1{)@HMOQY=}~$f zh84lOvlQlGBiLj%mn~--*dBJAUF0?%&3nqcH}eSI9lf=OkKmK}T)v!d;CuLSeo?V0 z(MnInt_)H}qhFq{tW-8B`<0VQgBqyDsC`t2TB?pwr>hIq)#?`Ypn6(kT3fBN)=$gT zDz$OiOl`5YR@<%}*3Rl0A&u}H45v0FB zT#a}qp`E(jw4BiEV;-y{G;(KWw8}A^N^tGrgZ_w?KD@FI?pb)>TVUbg3GwJ>!uO(_ z6vW~gbSJdqndq6==XeK{6*)5s@^#0etb|4cVLL28e2F(-_+a0DnTXNXg`dOmHzS_n zJ0Ed=XONCZ`ddQ7aGb^igoWFnSBZys=n@%Nct2hGHu-deP}cU2-a6Q%vlG@Sv!ks=zMjZe8x3Co z_uiK;Y$|j!qmQ@H;TYkEqxVRi*?Wwz*@Shx6uqHEzKO`!EgDbbN_nAspz9Ysw}ozl zJjUea<%KRsc`KqvNS(!dj8NQ@up`l)=DIKk_RS5)eZF#xus+y#WH@}gX1W-{I!p}r zT2Irp1!CX9;ixB9@`W}K7Be$^Q48Hk!Xg7u@0#b^k8@cRj{4AS-}BII=}<3q=#R05 zh8~5^(EJ+=*C5zZK2bkn-+Drxvu?L zT!#_0UFv+-*8VV#(Kl*s3*C0YuIm%ESn4$IG1{-i^C%V8g|pCo$HlV@ofeBbuD$hRS?ZHs*UP!GCAF{$(2 zm-d~Z8xwiDg|01hQz8$x(BTEAgA%z#>U{Sv_%x0Y7rDBH4$m!QdE`Q=^WCdp)Qynt zk<+ElcOAi~8&P$UW2DY^F2Sf9?N3COHrIuS>+nQ6nwJ-X>u7&8vQG;guA}{h$e0#7 zTt`@ZWMB&&t|M$}L_>33dt67@oQRVxbO&*MK@s~~=(gY(X%U-R=vKo9QzBNj&@F@= zjfj}vLWlC&&yJYdLMO@#i5T5Nhw|EYix?zzzI)Xk<%N!ouuEOjbDIv`hKQambYq~~ z5fR-&SBi7VjIc?a@42-{{R^%Qzt}>D`WHPh{CEo;>R-gt@I5{qOJ``O$>=&~HMxOA zqds5tL$7f8*Id4=ltLp$7SQt@KqzdvokfS6xd@ z%Q8@Ivpu{=YMEc0eWMtE_&2}i_n^7la+a-M;ni}yr0cejT4bqJ_W>=?^k1E zV=>Z=2qB`3mcJ|cy!$krA5zWk=9Oy^=i0av=RFNU)RzSBU2L3orQAl0`(V2~gBJSC zD|!Ai$$Pc(T71{}o6DICKU@5~yxH!}?_`s@`Q2&KwTzqY(&f0By*1_Z-o2*O&&8;} zm+O(Ze^>G~mxZ?J8AFPbX*`pLwMdOlDc$^>ME&}iU(36+ zT--J0qS3zGDKajHzbJkeC2z)>j@3-{8${If%XRpdce|3$cg>=%qYnPuuW`Y(Ex&Q; z&+YMB^;hbt*MeK_-*jIZC*n++J@ckqzSc)xfw1XomC=;9DQ=wIB(;p2>W;UjwA^-$ z)GxQ|#n;&5=Twt_ZiCP_o)RP_-+4FLkJJ^5974JhOSxsg^53L^}c7hN}i5nqCNc$zit0MFWS=0zV_1BMqjDZI0x}o>Z>QsYwoXnrt zZ-(rfe60^ZPjFX=DuqYXDSHsWlQ&- zyF9kkHKm(kv8Cm&Mfoi^U6pD&KO9e5knAg))!nA-O=pApUEd@(ZMzaT-F2^~ajuuH zof2BHy=Cl8HRG=}N)j66zbe(FYZ-e}mpSd@WIqzp3r{dkK^TKD9|2FdcQ@#35tcwk z?dpYIu@|0lTo^(;0$QawSbp3(ghL2;f^nyj7lrp84Ox`eyc|*1RY8`EGDW#fL6$Ab zZwjLPtAacR@f{0sFSL9jkGu{XGj5mf90DM>MG)uG6vQ<)4N;`s7>OA(Lr z$+iEF$AfYHBbr@9HRP$jw21#-2>+*^g}4_p5Ky;7=o52AZqoIdhx%fLejx(vq$!Ac z{Y#G6Jn8X5ke1yuD%#;K%5ADe4#1*KM{*_ zxYlu->zm{hJePjRpXrO6+B?x+=OVZe1|gIqG;RBL@z87J9qH56AQoZrF9dPy-x_9O zpE(Hge<3WwwxtLw5LP3s`=1Scl5o!v#CbHuQno|40bw)3_Wyyf3+E%wVL!qlUpR{R z1j6ax5Y8d*MQ^)zbwB*Oq%~jJfrx_A#e zgkpqBgkcDye`6Sn{1bam0T*Gq2)$>4&qY{(un}P~!ZL)F2z3bc2tvOFVF$tBNj1GF=FKVelh34O;%GY6R;PAiTQiXaFP2_7w@l!M#C(p8KYHe z$XfCy(x&F=%loPNRDC^4R%F>m^E}MTfyuypdmp-tzDi$H#_F%@Tl5e0?fOUh4*g?& zr@jl@jVmQwE;X3+;(hpSydS@v$MgO?LAUDudMiCpZ>Nbe7KwSuuN^t!JC~d_Yb7ls3v?GOjaXjO&dX zjLyc5Mi=8Iqbo}CXD?&U{T22qdyTE)IXssSx-f?vd9qR(ngIEpTjv`R;WzNg{=05jjf!MniEw!f6K@O*_&~^g7ywj-mImRxFUU zW5KK!i(?zvCiWrQp)JwYXm4ukw0dp5_Kvm{<#F*$3>C}X-EhrPIfF@1N*_sPYvWROMCMXlk%{7DbHKmSy*t&}TwC zC}l$Cgj`(z{|%U%8;P5Em`gdL7HWgFT5Xc{jP|W=(`)nzz(-EjAJC`h58~-h(;wES z>yPL&^hfoX`eXVmeU3g?pQk^suh!S-YxOtvb^2R+z5X_=f144CtMexVlqZxq$};^^ zeXo8{|6D($f1w}N|Dqq!|EgoI6c~wtAVTUvRG+WUN1ODH{tjx}8T|}V^mC|js-YP= zX=|7a6A31Fj<3H=JH{^_zXPddpp`!Dj`4L(+iUDK#`ulnw~XHb=`xK!Cgp)Opz(VI z2ieCT7=L*DvGFG%of&_A{3Yb+hroQXckg~RMvV;=1Z_x7R80)>h+L(Ty41wgbVtgL zIL^04q>`l0UgNAOtSQ7@x%0oqjxGP6M2+DJ93+`!fM?Ts`Zirp-=Q1myYxN!KHWw? z;#TelR8b_4;vInTodjIb1AH2Pm`~@A@JIPfK8HWapWzGnC8fRckTOkqSedRoqRdbp zRc0!WDYKNvmD$Py<$YzdvQ7C=*{+dB7LQtQodGB zD`&L`<33}8albLym|}=|=l>tkc{h_h)`qoZAE;TtUS+E}YOZF}0=3p!kk$sst6(iu z>!97C4btw^inS80LaWk-Xv2VrzFV81_t9_F`|7tD?Tthu$w)R*j8db_C^ssMDr2xQ z#29MaWehWh8$dmxex{IAl16e#KDnE`PCL_Fng_&L0WGAX={-Ol1!GSFtJEDh7=c9y|1SvGUB9F`~hUpE`T2C_TYAoRqgtejP{DmIu6V~?^G z=$Y5Bwd_sw&l}iwUdacm52%aPm(}I!Dxk~WRM)BX>U-)Y^?mgNV9mCwyELM8(qgn^ zEmcd?($R}&YEFHVzS$UWJVb6IWNhy57*c-6vc!31{*L7h@|ER6ohSG1{FyM~XY@ap z|3B3q!Er>Gfl%G-nBySVB24w^u1?Q}UIek9h{f^eAuL2#;!8K3)AFm*P5NIGuexgA zwMef=*o3g{s%^ifyr0Xn7iBdM2a!Gq%c=+3FNYKm7r6tt!-?d6QcEV0=jrt{7U<2L zbQj%6575t8C<_O6^L@6Jeavnp<4a5X}WR6D5AY9}B&Z%}Vk`>D69@oIlHK}}SX)MPb9O;yv?e6>h*sczMy4p0ZF zcdEr|iCU_))1tILU=4;_^JoLKfm*3nrj=_$wY#+O+C=Ss?E!6pwoqH7J+Bk2!ccTg zH*}M3!77X!^)C8NdRP4qdN=)#dUyS1y@!5_-c#?T$LYQGe){ctyxw0=)RT0(?$Gn} z0^OyLG6Ib@Mq8tu5p47}ZZ-NEw;BD6+l_dmzmZ_18tI1J$S^VuhmmbKja*}-aSyzV z$ALx@*tpgt3{RmGS_lig;ycK7q!jo6cJeUjPBIJmipkTUQmlS?hLpi;ejX^yrJzb6 ziC-sGc!F<};bbFu4=v+X@@F!N90pAWCj1C_06zAYz$u@i;baDlq(x){b~8iT{T}V+8_x_9Jh_?&G2S7B-9b;z{fkvHpg=C05?B ztypt&FWb+@@rmq9^v55w)BF?uSKgQZjUVIL!1{iPcau{}C@)Z=lvw_t(pl-xXDTVm zgM2Ca=Q@5|c@w>Hkn(}@jnY9mt9-9yDwotYO0F8LwpWIs4n-;>P=C58_o!XfIOTq| zuR2_ruHLPVSJtZ&)Q6N0)rZwOWv}|CW>Se})jFtoT1V|hb%=J87N?HZ`e=RBi9inb zSMS#nwOsW9Enmx5pAf5j)Hz~(kNPC~v8UAK@J#=#?$tim4r#5#`W~${>Xg6M6YqiT zv?RR$U8iN}*XyyG3-zl;^XL=x`?Xi$#XhRNra!JP)Yj{Z^d;JMyvZ%oKEb=(N^Ot6 zN?)z*6RUr;16cR7TRR4C^;7LTeV=|nI}fk*FWN=@sD2#v_k?~*SM_gumSBw?<3S*_QQeSDjVbtlb8EcKV^w*6I#=H6& z<2_@O{-&|j*rvZ_Y&SmA-!^s`yY+XBKN|=1ca6iwU-b8kzZ!qj-#30V8uTqDX43TS zruL=~{S%Yjl&kMH6`BgMF3Dwb>7Sb(H9e{yGR-&5*S|2;nKtW(P5VvX=-*&{kJT`V zl{!W%u@=XOHV-lnGCG?}&85bTVug*-#r&xGQKK7HuGoz~S~glfGWuBdTlO0%mM^T? zMyfT(?+v5gZ;fq+Io$S`ZLaw{+tapJ%s<&)^{18qtWD`-sqnwezn|qx;dWifLRyU2DBBxOXkkvr`p9MWh&VlBV^FYu%MJ|CBkOo-hv#`vY z$vgBGdJEY@d(qzHQ+g|Gbw6x%3i&6U3W}uD=ybf3&!9{3rYx+tAFTJEGzE6sKp&PC z`-HUEXJN5j=wjgPyVDJ@*#7igX{$S6s}IsYvxnFtuu@^GtzoNA;w}3rP$+wvJ;TD- zv+Oe#$vy`SWM8nqz&rXY?73Ll^H^!mk4bx8F70`pwC63b=doer%UU7URv*BX}xbs z>;0>C5Y`(Zt+#`;-Xv+gxv<_?%`H~vX+vPKOSE@krK`2SNy|JVE%OpA^MLNBAA|+A zmKHctTHq#Wft#fTZjlzaO6R?-m#NB>!W^;n_Bxl;rE0&*zZZdr_AmBp7wjg9476&m#}k-zp!&lJ89>grJdg|?fgyu zRR1)~Td?z&eRkeSj^f0USm?Tto1pCuQb`;}(KL*s^~D>063B%2i8PEIW|A!Qy<*ja zAKou=F%sy)=!3r;1!#qKp$zMDDnLQ#Ye$eaWF)zlgp>QwOJ65jgsx}{UL)Pm7r#z= z$QI!i*%tId-+L5C_y;+Gv8hwkN;0U8`eSuaE7~3_Dne-}xff$!;bbhVJDS`_Z=jvY z1dM>?k^AvxQ$${Z{ohTN(@}I3c~7pT*+eJNN#rB?0DX$=pik5JhyHBWGdPk>nig{&Dgho5P+X=h^e@MatMqY$es% zDz*ws{N7-1&;aq~LtC-8*xR%4rd$b+f%r?^ywvBD0q3k2JgNCtvY#)ta z2iV_eB>Ox2lHSO^V&Bqk?4RtPv==+i&eJ&2tJB{61pkuWinc(fw<#vYMBR!-u~H9K z^|Yb`mDWmYTCB8H+R+jvR0*YJ==q~*xpJKnL#xpL_ohRXTa{bseM&#&b~;Y!uk@!i zSPNvQ6BLJ%LnkVEN&%g$xD+>iP#K^Mpbsf`D1+!UtQ8tfrz_RUz4Q^ZHR1FzjH33U zv(@qHc>07oRh>%bpp{ubpTxSM#q?z{5=vL9uc#~N>*^b79eo4udh6&Kb+fveuEm?) zR{Ey;iTVj$r|wpd(YMs&>Ph;s`n7tR?$M}5>0XU%obJ>r(eNy zOrWRWH#+EP_>TGX8@)&`qTlLneKu)xbbYiwni&}9s$nLyI8&Gzqg-gOF@idq z1&L8#)<$>=teqH*Vj&of0_@!2hQ?ws0(=AOV*J7A#(E2zWVgZ^zhJ4BzgWIwX_oIT z7ulVbOIDSYS#_(K4Y#(k_GY83w_0yy^R0!}LiUVxqIDu$V6C-IV$WJ1us+6~vp#N} z%T`$DSr@X^)@DkJ>ta@Kt+TFSZ(G+{-(v6hsebL*yMCd5p=_^TgkKcf=hxA% zBRk+X-ETVk%x{L@5_ZsUso!$;Prq0FRvfXY=-U(K| zxrOD&pigJ8{#a-`qXke%SJ=N!Zh`$apxd;Bm{3gZW2WXfUbiNa0m7i)_y%~ zeFQM}BVpk;OAAky7M?0CJVV-crnK!lq*V`+R((ILI)qG;_WX#n=NTBK=uBq8o(sw2 zu;fu>J#4s|>=Ps0=-cn3HROP_oE!P6Nsa4=TyPihE5G( z$<&0g@pjY#pS%OLvW_f<`mr0?jj-6>tTzqBxX10V+eDU5gD~ci0b6we%Nr)fI%qhn zWs_j5uz%V?+W$?k{}r^mwEsA1`MufOu>4!4<@d$-`$l>j+W~9uC++-pY3CWR^KYn= zorRqjh!J;MBu3n+3w}W+^@zCzS_1E204?Jsyo3(rL-|O07rgzcbTr1=o}u?)UElk3 z9RHYqOl$crzKc$hEyZNC5hgkXt%HRwmaW4J@brV|OK2V1(q*!Bcp0rj6kU$>fYJ07 zv=Om%g>s{EBVDE3gf`-J<#x0Y8)X~u9{i3+=q8NhJxaG?+;a1wLAS*wi@T~9-2(QQw!BX`V-zd zJ+wisREN_>b(}h$>7pHBCbT0jv2e9ctz*}Vwu8lrR)gIj+6&eh?Zs)<-L#tYwv$@pzjJ*0yQ8*f8x=?NjzAZJ)M}jlpcj z-`Ty|SK3$Xe(hWBTULv?Cx12xo=Y1x8J%$hJ^~qz; z=^nH=i)34~R<<>7%C_ch+19)#+nP&l zWczXv?TeW;pk=w0`&kRD1w77LWcBdg);q0t^4moF!uyH#h2JjP7v5jAFWiClWdYB( zK4)FT-Dp|f<^!xBSau&3AewX!A>t}qJ^@#O2zsGvU`Yj)a7AKU~h_;45A=(-~ z*RQwV?ffafTt5$A=vVAl%wO^=_Z!NW`Q7a|mcQmV*>5s`%kN>o>AYUFM|?fnqm_K0 zXo>h?zt8=S@xS68l_=W|+YY6JXs48D(Kaa^c})4SitZKTD%Mw=s*J64RgS4# zRI#dJM8)iiO%)d^&6PbW%PS`%2l9yR*mi|RAfKSAUPA6B>Vf@06(N;oS$_A**z%K= zlTo%fmI&gAl2B@F`5Ad^7s{Vnv8w!hh0*LNSC02PO68X-SXDyRf{ImD3x03<56<3J z5mXUX5mME&A_TFZ7gwQu9fj#yZTkU4|1neO~H{fD+j@{e~EINZ5>qQ zsVw&f@S#;6NyU{T(Ka+u4QyG2E8LqBn#q;ZKr<^BU4!O8H(%B7=YrF$VYlyocUS=OVpx@2WZeaW$sOQj*DF{Op2{SazP=anum-Bx;}^ms|hf0hQ9jzGPsEggq2qZv)TEYF6G3R);> z3GB5cttwp$OKnN(k-lu1|Dm)KSNEUNL9{4nQ~rfcHNPufxuo;8sq}0~=h6$M4c8*H zR9BN3tz0u|YK?wn+}7I~{Twd~Yq6zo$z>{wE{iRTEq9dXmvt@c0qP?uUbdnsXi2X{ znPs_UW6E4*gUZUwhL(+lTvIl=Y+BjU--Q-kE@gh%%(6L`Q_V}Ce@$)k9M_gFmtB(3 zL}e?=R{zJeu54Y|2GC~EcF?Y0rt;vHRJOnD=)a{y*T`|AtT|mQyP8fzbFS>7_ro^e zm?XX?S<8P93T$~*ms1ht|4N<8JC}F+7wTL7>!@dodc5-2pv15$rpXw(mc%s0ucb0i-nx6Y9AMKuNav*cb&WKMr| z2=+>pc@~L0{FIaz$oyuR$6J1((6AJl=Yq_$N9I|stioP4A*+VaE0-j{S@PjtSzUyE z8^p1chlPg9@|oOIoYmG<=58urR?@RcMS= za<4r~CN#InJR9WoE|gq8YoPkX5f@8-NZyAea7Lc-J znW7}j+GxZMvS7406Qjn!70TJ6e9Q|KVSXr(2#jbDMv5yiBUFVMp|)~H=z2L`+zsQ! zD=_B#8tCTB^Fs9)ffgu@o@5&+j%+8pF(Umb5F7pFXmX02DN2=N%IR`Uc__w|Lor_@ z=7{c{uat#n>_V z3}eTw$w7v(WAcRi`di^-RArs$-cDf&u`6qE18 zOcA*xM~W594%Jf?=(zVVcO>S8EOK7R%C=*q*iX(21<3K@R&u;J9OK3Rq|xk0_9ML! zsO(m>iySHLCP#{U$&upTyo8t0cs`g9p^1D9A48Mnh%jsrb4O`%T-c6r;Z}4o=8J;p z-53!LrlXV)C4^SX`Jz9`vEZ?oFS-@8Lt>_ArkoL)B}aj0%TeH`(F9|nWEKlrsxeZ!b`WRFRQQ8-Kv-) z+Arsb4#+v8&%|giJt*ghK9_Suhs2mKJp+Uwr9a5op&!M_FTDh`UsZmgU&x*g-Tb6Gsl z&rh>teY5@nOV$6WpJVCzdHs8qq5q&?WLf%824mSkFKev82sJ`kA#kequ_7_s%Ut*l zY9Vu*pEEzl2FMZKfpSJ@u$&PZB4>o|k~2bMF=D%jO%!9b>>)84%N~<6KTGAz&&wEx z&11{`3j9251;$}3*=ji#v_{SatrcUh>=QW)bU=)yvd_e94?8Djd)RkkwuhY;qp0kH znC)Sh>#_a7E)qs>gKet7OqpSq0?Cc9-(oJO6=rzEoUe+Ro9kh>H-L1^`TPO%yxn1Q zX3`6DzBbYi6a?E&fyJf)D;x}~b^v|lB!e)YD`tMWz>@F5o?|fcbE~xEREhRTC$E6) zG#f#(wFdxgQ zm9C(#Qh%U7UPIfk8WadSe}}e~Sm$;?J8#9xjh&!KAoq6DDEc-1hDHl?EWHkGfl9AO zTVSF$F$?p1WBv&rAn z=h-FJKwpqEPD?T4*!iw6Fbv4flcgA>-jDG7Wy7AD{*ua zR_pYko3UaifqsD2CzpPNcIQrd0PRi*{TnYs>vN1(^J@Bc+4>yEY}G^bAHaz}Lcb9> zae9`|;`-vKlJ6g|(M<}cD81jdnRKsmm`3?Lm>F%yuEtFhi?4PV3j_&UCh z+4v41-Te6(-oRR62Heb|uYP$F3;AR{}n7$shbXV)u9N-~R8(v&pT z8ME7Y>_)U@F4jfNZnK-j>^AGF6e}gH8`j`dvF>914eN;&IKx?-ScSuSW7fNx-KvaL z#xb6xQ5FwW`x7ixATgO8SjQ_is}<-Bs94OV_s zeq=+GpOl~2P^D36WOs?Sl?_u3wG|sJ+E!Kryk|Q$L12K`M4&yR*#l^8JFzK1ea5f{ zf&9FIO$EZ|M)r`He`C`y|CY)gR_&^tJ)vf**=&wLowIpBg66aNn3r?2XMhDAz@7sh zw3ICZYWo59B3kVi*;1@R$#+pxF0Y{H*uTxM=g%~3v?j|Mo>%Up+Jpi@o>$lFL z53JzVYgO7XehW}aBlxXYO*NAD#fqwGej8SXjp6Y?vX1BdwF%lJo&Y@S13VQohckJa zHcOkubG6ypY@UaeVsm)DHdmX=3xIQ4zzczTUC3QPzG9vY-$Gr?J=zj&2_Jy@#?^eF zR;R7uRaj$H&xc@*)doIX+o)~iBehN1WQQ_V`ojKvvB2N*dLVG^e7%^@ zcO#`n!q(l>a70MUT5#(XeuGi=yk>^xcXfRv0OP0uGGsL~g>j;>0gn0-|WF#DRS6arJ6 zsgwaxoU2qB#YV9*%qTS~l;K!80lPQuG44^ui+)|fVj}}~k98>bE45gQGFh2~^(YT1 zld*=;s!YYI%2vul_-cF`Wtypjse|&c>3Y-k%5=+UOSSR{dfdIT#|=XtYDQ0K1%=9f zH(d6+;qZe4@m1Pzgechu>%dUNV4uz)6}_*(UG)H|@Opcq_w5C;NiW+9-?uk>>OP?M z=xuLBe=M+Bek2|giheo?y?ioALyw(?o?fi*GfS_$HG1I;=(9k9(l^&6ik(XQ7Dtbq zhrT}_#L+Joz*i^)S+mp^2hV&~%{JK@&q`AOp{j_c4J;;dp!K0{u-i2TwQ`=oAkB z1J_0G21cc|SWg3uK&nvbi&!L5MZp>yd?8+XBsx5j$)pvqD~pIu7X!7TOVo-E@8mVe zuhZ9wO5dPwKwd-FAm>{8CgghhE?nUE=zHMr)Azx*;Y-sR{fK@94zC4A+(mbR@1y&` zvF-r;I6Y2m!h<0K38P>YsmkyzO*9~Q1;uI~tnw5qe!vCF%_6)RV&E+&7;*^1KL!BX zij{xDvjLA}QA8E~4fGw^4UlhSUBI!@6!~vuw?f9cA#6=#NjO$A!?*K+oW(x|2)_p$ zeh;{KLox~P2Qs`L@M2aBj^T1}_(0&5Y%usRHVnLmO&~TljXjL=A7PJx!v})?3APMJ zd6~UJ+F{kv3TR$suR>mlH!UVs=|X;;p`Ktp(wpEw=0Lxmt%tmUZ2;fMHsZ`Tu}$Fb zv-fe0AFvN_2evY-kriGP^xN5XFKS!5sGlSfEij?mhe-@O$~a;5EDkyp}(JqfFrskpQgEdjvAPE^v5V(7@{g z$67sTp5jm8xWExX#+nnz3;9CG&+%oD;fI044+DoE1`a!U@_uDMGOP8XUeFID9qe;j57Vd_g^s_z7fa=Mxh4xfw!!Y9iF&r-9AAv`m1cxKoN#1t`Mwax&@@XWyBnTdZ;QJ=?a z$zpXe>i6J3s6T)=s;K{XFURl}z7@|Q!=D3(KSvDV&w;no za8KaX1w(GHg^&RFcA=2Nv~b7~S_ILs-aV4E!m6VvC!CVW2dKWKk|zNglc z*x>CYq5Nbm8TwSs4xXu@h0(IKEaV&6No5|ml0l|!!3Dj-*B zRp7vk5fj!_4uOpIP1rh28wMFGEWzP5;;i5`g2QVBAFGW8hwq4-6SN7)gY{65Yqd!z z2Yw{vXS8P^FVGf1hQA2DNJIZ6JVj!Gr}!#S@GXGVSWjQ4)q$_k)4R_3HjqI0k(_04jnzFDr+Hw$kPTYu6TNr;Zl6eE)848(Y=&Pkx~DIvqB1jo8AaEp%d0(_gn zN^I~fZN#JlDF+$;C3t&196VBwB)T4@cY@qm?~VL@^xN>&hkkm0$nZG9;c=5`HIS_?_VJJHf?@ zbARD?5boHC)^`(C_@<;SeAA=g$MtVWuw32lFITtw3$K*;%hm1vSlt$g zQP(y`FnD{TJu&dDi4fu^MvsUIzG)2P>y7J)&4|UtK-d|#rqaW_`}8dxni5_#a!g2SUF8a!J3 zi;jV1?>OQFV*)hrZo%Q*f=@A~fQxTAn2o7ed94`_Vg0T6f{2ww3V#1iT5%@QrTGGLCAN~&~)bf17Md!9A^0&eXy%yxJdc{Y2_6&-Q~X4ySGvh2y*Q}SKQQma#s zyKU(nccEiYhLYVav#0w=R!~;8W00rZGubmIyUueoabn`cq;albQK~1m=+G5PvS;}y zXP1{mDNW=U|T*5_BVn%G*XD7 zDI+!`Hlm>=#aC!I2+GUvE}FErd-}US96hW zNzU$x!Op!+dnG!Ok`f~lJEfgYJDu1$>v-1j#BTPz_Ps@i1ox6_L6g)xzP7e`jwac) zAhBm+Uy(mCL0pZXM8_50q`Nv!Ompr{bQB%Me)&Zw5E+n2wJeGJe z@eH2s`NT{1Qw}9%u85OZlHm$YvZahp3Q7t|igJy1jZTVjZ*mNB9&zqW>XOtwse4*^ zQe09$@FdjfSXV@H*QPp|WKVJ?6=vC8o$RNQNbxRtN zG@|IJJ1J$OXM4&<@Z6;8)F^L_gFG(nqGNSZZPL`V^5oUYtCMCV%@A=~x%b*UJ(6a7 zdU$%cdM3?t^#ot&>M7FB!Pv6I(<8kS@$#fqGOSHnkDPszHi5P!?G(X&Hfe9tk)(qN zN3x4O>pbhyXC)nXB_y44ElX+uT}V2Ma6!sQCo9SLjJCl7VyCy#V_ zT*bIg*5onCHOV!|Gt7A~d2;eJ@R`YTTq|9{$xD;x3oE{w(uZBG@ut&ledFUbR9_Ej+O&$2k2<>q2!}E z`*ZfYW85(*B>6=0=@c#bT=K;flA@(pQvzL6Qi4GdUh>2j9YUMg3Dnts+%YJnoBeo7 zPuKL6%9OqXtSJctNJ@SSN=tDdT`UOOWZcAEvr;^uVs|Yxl_|rrj;D-H8JiuEGBIUJ zGn#IX%W!4c9Imw3?1+qjlvycrQx>FLN@cEPUdpadS&XnOy{}I*H)Ul`0Jbkksq@TF zsdp{2U%k+mg&*9WLGJ4;|*#x>p2)jU{Jj9+w*$2{Ar>#rdc|dx~KyNuG3k1|;2z`vr1`xI@wd(*s59CD)Gh zU`SE!nDo8~5$T=MJ0snt=n(RE%PMqtcXvJDXaXa0G?!xpP z-~-clr0+>TkbcWf_I@r;#(sN}+-`T2+nx482jC}OvJdoJbkDX| z`)IJe)IJzgjWEJqow?XP&Jk~~wNJIr$nIvJZJ+0~lcvM&d%0L7hBy6-3w9TCi^z~PWL>K_U;2u;oxODVn1#_Rdf{fW{Lf*dz1Zw zJIUSv|80*9u^DCuyr*kW*K6pme?a>^U2Z8_UxL`Bcq3VNk*THc*hvW zn2Z!qW=3v?D`Sv*c}980(2SAU-Lkv6my3`wM#x^usL7a|F)d?e#+;1#8H?QOGnQto z$XK1RE@MO1diSP`&8}gtVeU;$l(9WySH}K~LpcG0GL9n0iOilEr!&rFoI`vulel-n z$JIQGGOdUMGlMfD+(()yvs30oLGFV-&B4sh?t_`#pzE30H!~qK4Kys%k(r-4%zZr5 zgHVigB~nK+M`w<8pF++lnbR|8W$tjz6>YAM=KAW@+04am-1p37nJY8vvUXFS7CT-7 zSnZ>wj@8KHa;#{c)3Msg9P2#gjt!2@I6}E+kfd@~f@8a5JJLfPyFmNJ(LK->9rcWK z1v(Bn$2pEVPB>0G&N(h-6TDYx*;a5t!JaYM5wca7oZT7kPM0aWlV@^vCwCHfXV^g+ z+UUO7j_d?Tj?3i9&d>J13!IZ(nLR9fboN-!9MDXJ`Or;>72J*)8!wIaLj3iqar0fb}O zCq0|9&v-WAO|cnepBJg@OM*LD6UkJNGstNm*bss|M?KpST97lu8RFUR+3$>U#vm5k z_InOFyC6Q{?4C1J_%4E+al(J`rB34^DFgU_~m)X?HrEh0cKhLsvTo zJ4b+za|JqUol~7l1_TZWcFur@zYt-zbDmFv$M0P3T;*I_bfV~lbG>uDj5j&A$=J(x zI(OoYZ?E&X6Ys;$gL2Ce=P4OOC*xDjv#z<$3(kfdCC7|5sVyiB6rB^B(>13@PM@6k zoRpl*oZK8&&Y+y~oS``*bH?PG`5-Cq`#J~Y1`FtW$xJAVY#Do$L1NieRC6X({df4e3a_BM#;s< z>5 zg`2KW?ncm-+#R`lau4M76O?-(_pqd6IJ1+vXYvfhmvYY|ToQ7gQFtMbA!d0tkRdsa z_X`yDv)iKb1OnPv@TlT@-(}jv$s7TMGgUf(s%FI)OaRs9Qnj zf^HyBLC=D|1qlUd1&)IJ0#Bi>pt$hD6)Gq$nD~n{96$`V|UFDzrmaSU3<1R|Xf3D4Z>6Tw!(LIMD3E zTA7|&xE%5F!Wl9(uW(`E5~P~h;V;Uo3b*-aE#ggu>zn1?R=BgMYhiWPsKrT=wLN3D4%QUE{ z9P&t+9$7S|s0K6#VRF&5qM4B97cDAUTC@VRx@cX|hN8`|@m)pxF-Cb5{m|*6b43@$ zxSJeh3wA}gI=MPyGz()-7=`NVN)Yc_t~8g!l`qC?#3+qxw0o0ltQa$KO>s>hKwPsh zQnvu(3>Zg{Z}xSrdc1k#&Dym`yeW&fVb>YgdGRibX=MX%sX^ZNLw6Uv3*s%$`#vY% z-Uhl$-GkjD;LDA3*Se>=XSiqM9cCWF63}w@D)(A9ynFXH_fGd-pErKoead|n-m~<7 zJpo>S*AwgMit)2P@K509Ne|BjuMEBzG$TD@;9E`hO!N6fi#$s`D}*N_y_M~rUDCTa zB>W2LM_dG)0v?0#2nIw9=!ALdcEBsBKs^Zj2+#?jFrXa#31-VckrFeaN!%uvxXt!J zUv|Y7fyr(o@gp1v>YITO5hxN4H0Le&f|o#%_)8RtpG1+EB#I=1-7(LAOXO36)q91t<~+uq5JJhziLCence#KSF^XDF8yGkQ8BG zfgDjJazrNrIpRfu>qLAzgpom@Fo_(|BxZz5%!r>vi!g~6F-x>au*8WN5*rdEu_59c zWlM2y1VSWIB1AMGL|()73WP|IM2Lh-gh+@)h?pcoBvK+oG>H(=BtnEsgb0@i5kH9# z5!jGWAUncCN5KZDiY`}xO2pq>Ux{QPZ4}$M80kOIqniX^f@dIx4Rp^0S z1qX5!TkB~(%6yx?gJa=ay^w)l1s6yVo4~Jv1HTGPia>ubCeR;NiT+@i@Y6{EzV>PW z#b9D4Y_%}V@&O5AB^~koS3hWgZiPnRK~#wZVG;?V0tpfhd_)9`fL|fd>f?-4nP#g4lor8GsT7GW>5l zyMx_@|vwgFK#%hb+({9EjOk>^q4~ zLOH#{X zm-rD?;zyXoj|52kh$8VL3fscAAdf(iC=x}YO8f|u_z^|oM-+j`#aRK73rvSVkyr#G z7d)7UfQRDS>r~=LsKk$0Bz}ZR{0IYnfIHkSRd$f(tB( zBC#Zj#F8jL@IDF6Ts{{$y@-;h`O_$2KA#Vyi9nYqK$k4Qv7QCSgh`Bv0*uLG$S?30 zpkKmYg!~eJ3Gy;vPng7>FknxB-@zA8UxR)Xa41aTP?*G_Fo{E90uPM)CU7Xh5{F`u zIFw-Vbu6Gi%!(Q43_ryW90+0X2qgmekw_&Hs1t!wQ6);H6>!4%nwf&B3Ls4|j7qpb z3!?;}g~0_@g$lGVu}GAPMWR%g_-;Q8s#2*WHhjIm3i1$T2;`wauP}*TVL-2pgggou zmOz0eMmfL|lTd*rCar-bt_7c@Ou}*Tl^x`nq2T|^#VQNPk1LOZ&sLs*M&Mi+a4t_1 zE57-<2(m!9Fo85i9w3cT0+7bw`;>ho)`wv^rX0gHdeJMu8>1ZHjlqF8#=Zi>!X$=; zN(>7Xcw?Y=fH%ff0B=le0&h&(0B>xN7_8{9LH1WM^REV|tssNI1?EK&U-|)$P=OJ{ zmuG>Y6KG_}0{@~k!@n?ze_;~;V#3#GvEB+_qD@5}fqzjX{zZ}a7Yh7K4z}h31!I#a zm;i}_QGkLef(GBM#a3XIfoc(m7^_6Ys1gxl01@*7@F`2wB|y*!EQ|^)%uC?QfQX?I z5u-{(j3E&*szk)7K*RuNhqWkgA*aB_bP}j$$RDU5fNxQ^VBf94$fyz{!zD&Wl^7W= zF*00YWVpo0nA8iv$lyCgz{PNZb_VW4pkug1$Eacj3-BXBS`e@y_}VwHDnLGi3v`SH z=$J^z0t=%^EQ~6#Fsi^mllB7t4Bk%z&Q9yE^@l8QFF=ZXMZ; z(#^`s8d+0=KPO>LjV9}~fwHDXaITvOxXni~1~NjI!R!J2KLtf>jXnwlukS+0E8 zDR31io$)WGLVQP|*i{TH!O9vhSJp&uWsQ?7Yc#H`(YUfkt=U*>6D41W0B5cXUu?&BBEARu`>yW;W%Z54 z)i*A#zHxE&jf<;qTwHw<%GEb6tiJg<*3RJFxjbtMkp zr8t7Kc*FH3)-)Y=9S4oo<5=B;_vix8;ae0g*98~WK)DsSiq$a~3@M%QCO}Sf!@8au6$5Xtz6rFf>j~w$9v9a2#DkU!Po8Gby(e_a&?a;tJ9S(T-~E$bAa%Gz_{WO#TP1vl}-p;C|g<{-jLHUvSC*GoQ4hl7XNzxHvj(Y z@a&yA)9U^8jg>|98w(2R+XFp|q5^@^EydGInoD*DdNwF!`x^R{FK(EcaWs8I;6ipw zVMWpKvYr)D9U*hyI$NP#=jcQe*L#5{=YNc!=D zuGpVK>0ApLlNv(O&!%4}+n3Rj(UN{y{$`sIk})-7s=p;;YT=2FkP*qyJtNZQ;K;xh zjt#sFuW&OGvGtQJqbMUSBL`HGSPF)Uj7k3%YBTIGDPs^M!?=yi7?aT~xsu?Yg*2p;_PK8@il>Q+3T}66m2QLlD#>5XMK$l9Q5?fsj|9QrenRl2e)EuTRMt!Z5sH zK~7UJG;%*7XL8Q8+++!m%?^ecvainwTn=2$nU}LLH#uiX&hng9W#e+zmTk&u&DjzR z8?oP>v#V%%&fc5@Ifu(O`B#?C%Q=>FGUrUr`P|MqmvY*3+H)Q7!e0z|xt@Vtb350U ziy;00;lTzxz)J^*s4o%1ar%`*_XBjhecSRqHbt8>@Y52?`Vhv&A{54YvH zo50_SZ8yRW;=bH{w$IN!h;StLNJk&=<$dn)+~dN{J(YW^Lz;WGgSxhFs&CB00GnY= z?uERN+{>U9Y(`#q!>oo`dEN6O^St$w>nGGt&P&Yemp3SHP(eXn8umH$GwP>x^c&b0 zVXKhiqDIlO19WERp;ycsS>etblh>R#DQ{|COWv%!xs^qE3-T5tEX!NgeM|Q(^$YV> zGOWp4Uq26FN#2IM&3W4pcING=U!J!=?@;}6?2lqQf$enOxx9;cSMs%dcfOS$f!?@B zepJ4%epUV2{G|HU{FM4uUu$*Ke=iRlm0&2AiY6Q_va52MW3Z z4;S>TKP*@fQ;<-QjL4~QhXn;?1=R(Cf}sT?3Py9AUoftCcm1*YlLZqCW@2n2VG6eC z*k;zBVK`s1vS3dA`TFzVEh<=Aq7@L`W?Q+4{f_()C3yvV14RkFyUT_MW3k8?)iG+|sqhs_D=y@%~2vry!NGJ?193&x- zZ0`%Z2T}_oZM{No>8`@W!hWUOB^0J%%PA}hWEED}p>*Eop|G}e9xMziol!Wla7>9( z*jzZNaB4$j;oQQO!dbutMNx%|3zr4T3RiMew7yWn8f+b-r1gaxaI_h?0leT=6oq)3 zgWEPa3Y5Xd(ZZdDdkXg#9+Kn2qtrT4c&_ktFq|vASa`)hD-Z}&7imTABC9B(s0Xxs z+!~9LiigUGstAJ*gcNLYWOMRxR8oY?2FD#+V^Je`_SOh~C1^jk;n;?7Z^v#dYATw5 zG$$8LE1FR>yJ%j~!lEUCwxZ=ls{(2WCo;een)#LeD9l zALv=U2w_F>Qfw=VR~N4f%qeaw-den=c&n`$m|wiRcwh0s;v*%I#m9?JT?-eAFBG3G zz7SZ1ZE5l4ie30CsN5bfO2SLJmqZ52O1veBCH+d$N^(kyN-9cfO9lm2l#DDHhRw!i z?3+uP1FHiou&omsVGOoOfwq#VB`twXfvtfZB`qbh5ayOF2+S{8R?)c?w6M?DP2{%oMBbzTJSb@T&<;BKpg}g3G^)8WnYcJU8Q?V50oA*J%&Hb8WKuR z22KTz2TqlqDLs!K`RwN-xw4+6?WLE>96*FJN0|q*&N9BcE?h3_Dn}i$K*f|LVD6Ju zmRyz!%#uImHYnwb%Z65<;y{O#FUCvciu54o2l(Ok% zGaI^>%?Sj`=9eukTZB!*in7&Z>&n_7*;KZ*A+jN|Y+u=tvK`mLZX8QE*x&{Kc-g74 zvt<{`!^KRn46`QN0xib6U&84Yv_jnEYB$~Dz7N7Egw`q4D-I6 z@{#2uu`dEOro6d)5^|3|rNO|g>+>+Hd=`)90&#Yj^;Jj!wXA%lY#621HVi6XQ@);I zTlt3a%@s2$Fcy)}FtU7S!?5!G!LUd6m~kSMA0i$tKUaRD{B-#_IjYdgFP2}a&=A}e zR>RbWF%=PC4B&Ny=4;fXiXNa_Fmv-&B-ug61fPeBl!mzsiyQ2)pdtg*vWCSKd8oU} z3V+3ris2QF6-^ZrDkfJE#itz99)n8#_-N1DN+YmM(h>eao`v&eCco5;pz~ci?4Lm#W!obUT z@l;5q@dX>*nd`*LewBkN(<*Z+iz+KBYe5gI99cQ0vbpjQ!k%kkO{Iio*o3#Hasjr* zLC(s`N#w34ZUAnm++4Y>awn+$v;$jJO_ftCTPkN&&Sf}MolN`7U}-Vzu7tcJH9IXk zB|9~nBebOPH7QgctvrEny7F9=ukvD*4`E}~#>y*r36;BQdF_xYf0b1gQPra=3XjQgK!?<5<+`bdo?~(R^7GQQ{5TZlOdrxD;QF-PpHPguQ~=Hp*r~sk}OKf zs;jF5)kCY7R*wjVrJxaN)>n_lejJY`0;g2Zte%d|me0X{e)S?At*Exc>gv_i>#7ej zY_0xcXyfrF!CEV*V{6*Dqk4DsKJbrJp9+TK+@GyJTYaI%s4=Q9*C_Z?E2x_An(j4` zHEA_ogtRY&#F~CJX>}88a%zg|Ce|(lRe^0-O>NB}U`x%&;MRiu7;Mc9Q)^GwOsbiR z&{8w2c2&)+ngum;5f;=eu31*I5@AoxnrmSXj_j}=`wcalYqr(wtl5Kjr|YKI?5{am za|qjsn$xvW2t7JN%|(WDH5WTLwLQSOf@2MMMYw{bB&HVJdf+Cg6|@yq8*!b@s9Ilb zQf&%uN9EO))K=CGLGUBUen{w4B+s=ZX# z(+(X}ZF`X8;NDZ$xvndy&#%0WE3m9Cp)RH_8P{O~Lvmdz=qz5Pa;4g#;JTRAb%DB} zbu$@807uu&uNzl4v2IG;bR5k`m~$=6uUk~Nv~C^33WRlatFg7!9k1I|cd%|Bwu5zB zvF)haeeL)-q{r)y5RccLsypRZ>ds=jP6We3osq!E4AC-GQ z{?q<*{)=*dNABv>M+C;z_mDd+^-1+94GZcs>htPLaF3!Ao7^uLjv0HC%)#rY$&4Da z70LLgLPjtR!!QCF(_oJ(Cdqh$s|t(X05MGWd9|rdqcBt5Xo`C)C{M`cS0qRH9c9&s_ww1o2c%=&^!syw?I8KkTdt zX@`a1396%9(ZK%rx4U|Eu@lc`uEx8bR@5sOoLhl=$ayVffyBjgAhF;ObWSBgNBKxJ zmHqO>5z=GC>BJ=JoFt0n*Fx~@8Fo5LiegzU7Hog@vt_WbRq*O-q<8l!pG_9T7qtZY3tnAJj2D|%E(m>PkD&6AVEeB~hZAok?#CM#kfP9d)1u%8 z;xQt=_>1St_@XSnwBiX!{)S z54GdJ^Xi-J4+7t4Z-kVks^uzCpl_lzgP0_u7j*Uz zsbjv-`fNc*4f!t%hHfE0gBEs@epb*`F6ithd>wDHK#JsRtTpX1IWP5}#5ak*5=~9x z`JN{}O->|{ZBKnmXw^@Al>B>%ZxF+YE0|U{q4Abk(N7}%Af?0)uF`+?MxwF5)AD_{74YF zl31%5>v=&no%~0L3h_lj^{OD=xg+T&liwjys9>Q)i@DbL0xGue6bx ze`dP>qGmNUhf1EctCT({9QA>#mjs!g@uW*gzef5sT41lL4k2dle3)m zFB9)0{+W15P#-FYv@o_&m#FW6lSrv4V;J~eDak?&qX?~A`!>NV7 zIV}3xRmOdh7*6SIN-LSGG339(y#0mJ43X*uVOhDRG*hYcla8K|LU-j%A6BrOk`Xxk(G%i2o2B)P{{Vmb9)1A^Si?0N{Y0wY zPCA|Z={%R|l8WA){32R-;;Q(ECn$Y_d59Jo-7ewn}LIeVZfbV2x|r zg?8T06erSVJgq$}e03`GfUj@E8cR=0pyuo3zs?l@Kxqmsua~&mdZx~+Pg^g1U89t3 zM*F!)^##IF4v8Iw{z2`_bnTL)A5gB+K0UeGME#FhKE*<-^dV|8)A}cIB>8=bix_tf z^EOzd>eCWe8_X8E zaE*P3I!Wy4^c8w1T0TL`f2V#WrH_dnJ&iVN1og)Rod#)oLiIahSu0_g+$?;RZBKiH z^sfaWg&%f?^X#5wjXW#%)#FU@D9`06&*d#5uO0OW^WbG#2Kiw^htvw~x|{YNpf7!x zd6+;ueW=;z>MKb1CemG)hkJ;Vh*dX4Ya-xWR1l9ML+rLu(P3qi2 zyn#ArY!sj*88;f4mr<=j{0rp{0XMZ`LfP7qgT>e9rT}s?7QteLG^_{eTr^M0koxTJ@Ks^q|CpTdOm`%4FOmLAeV?&z6&Z)aG41j9k5d ze-NqaGi(V>R|~;!Vk&mc-bMN%lx(yjksG-eN5U6YD#=B+nZKG={OsCUUur4)PF?M)$I{QBhQZ4 znmUEjw}j((nsG0(Bxg|PEc0NGk?v(Hy<7OI?FqXwE=OhR_sO5h7Wfg%WFSYa1IhUd zt$X5AhaG$sU`Zja-|&BQfDVEOeEhPiO~b-2N>&BiKUOB=1}Hy z1lwDK=&K_b_f}e7B&fFWT5e;VE+=O^ulMo7(Zi@2*^YJUI{l<(kG`Iu{tubTINBda z%UQzFdfLdiJy|cjho(I)96f~6UBs2dpHh=vNPU}SPCmY~{U&nA@tE399rpEFC~1xv zR9>4J{eWhVUfM*ehSGAOpnA4_C31B)V@)CU{0w&~F-jMF|FbuM=`0_ESV*+{8hSR3 z?LmE+{5u7;VYFXBoJ(wGx-XDU6;vOlX06Q;sX{-Xu+1oE#1|>69CXqo6*YnQDN|G~Z>D~C zYCes6=g7gKP$!Z)t4On7Q|+FPQiVQ58$hYep%>Ne5RN1KvyDJaP(Mi=YGeBZl;n?u z)+UqV`3(1~waHhXg65?52%+13pf6BoK>J*91_(#1B?&Zo|b ztF7QX$P~Ac-$GnWOd$TUy$L#9h1M-{9%YJ;GR1P6Bk1yzj-$W zVc}~(vN@zz5|>f`4dMZE))Kc!EH$2I^nKyzWyGf`ZDJV~(wbd&ql2*|#ox%8>VLI4 z#C6nMC$w%``Zv6py)oYkcNU`qu!_8|R+gJn zXng~5tf1pbO2dfXBSsK=2s#u=CB;7`78?8M-`YF;=h1MVA8AUSx z4^jFIW38|?ZJnT|wBGwk_n=NCb32%t_mZ}+np)bQM1C!?iu&W3N;5g*iT!z|{i)wS z$iZ2tf2C$Jv4nVoNHK;PgtLn$T_Uu)^RqL+PZ{@CLFF8!tZ~dWB-U0+Cz9?ZIZ>_% zs_o*XwQ73rDeN)d5!6RXKca6C)c#7HKM;qKGnDjT#>!&9=@Y)1PYZ8Qr&3UFq|SKi zXru>Ihj*XU7}ECLj5^Y9(eiL=_9Vxi#qXp|dz97ns@&U|EK=06q&tTr9L}}1PiSEt zEu@K5;e0{4OluqiD8C`ik&m)LXyr+2a4K)@={v+lh9hcjpRH@`+XVf753o$ zgrjUBZI873F^4^V{ zoFud(L{P0Gj#74kZjvX<+QagMT^qsY>~o0^5!?C9{as?atC{v+1imXz;nj9|p02jL zYKbxqY!BuE{y|TaboCNqSLWdovD*1-q_v&)|1OAc`Oj*zoiFl3C_4;dq|bi2*;4f!MJa<-o&p|yVG zR0~HRLp%3{d;op|>3yt`HaSzRjXC+0I8D&S)fcYGq(8DzH1z;=whKDy1=X=E=dp68 zS~s@vZnW8rZLb?^xf{#pA4HbFTE$p(a#gCwXs0jBEu5SyEIl7}d}3LBf$4ri3xnBW z2eZWvW?LQ1R0gxH4rbl`Q#AFrY56Ih>1tY7&Ag2zeHYPfBhPNR(CTGbSEF8Lx$SqN zuJyev)xlB=`fQe-d{04D7m-e3K2w;_-#Tyy4^PSEyb_(VqGKm*{+8#poafb@`ddg( zWUGi{u5M#a;;3(zRW#4#1M+Vq|9;x}gx=s2@?%&gZd$X;XAnz%5T%3ICg-#L=*;bT zYF;A!2Cs%j8;9pTxIFaD@s-y-NN&q@UnbiT}AnT1;F( zX*yGRLRz)@6!YLD=T4#3P0aZq;#A`6M3#)sl|@=3Iq59b=`7XhQmT4)@{0u3C!{>_ zypzatxr6+Vcy>$3`Gh68Txh*j@}?hU4(;|liJV^vI()>NDWzZ4**@{r+Xd*CU*)-^ zvHa6$Gc9Bg__igAuOwM82e@n+37L+?wcpl{rS{@*LbpUg< zS4vFzJ9B+29(>>-xq2coHG2x@JD>K;n>P`ZRQ^%869C8;UA z3r6&xqh>ktz*S+|b3Csl%;!GptYXgJ7j#S&bkY~86Qy)l#UrUeeW0GwyogTH)v#>7UyiL3|}p?$s=ylHU>DkFhk^wjDKMQ-6&14brSH{hZM1cUTMGmKKF?dg2`M zwNSxbf_N8<=sZiEXPMSGUQ@@JmVNCXm9(_CL{qy{VyQ<&s(YE1ouVqV%M^4sQS&f+ zgu{4$2kzC-!eL%b4=}BVZDfjl*jx3XP9K(DA9~K4n9nX!4%$7$Nlg7NsXMI?Q>h|y zJbNs9&@9})kutz(?9_Lr#T_OojcU!*11?A^nk`Op&VJ4Vev`OMI1@J!q$s^gHSlDF<~L%VZL3iY;7A zW;yhgbJw{_Q~MLoq6>50TlBRQmdr^(ygf}uXz#Op*z>B-(B?Bd$8Yl-iV*7cCIsA;+OuSvb z&aOT~TA}_Eq;20@LQW;GN*D17o>4cJ3|FhFKcK#kxLeLeoy;8G$uqi>HN`Qo=8&9d zA2L5S(wk|&B;O$x@O53O{d1%_zEVFX=dz$4L1g`^E6Kl^XWEzeyrigd)UC~?^m$@5 zbNge`l(m3cVm7Y5LB+HV}Kvin~e#18}=tU8-Qbo3#3$)4}?}$ zhZH0K)#O(aUlr81OIo@qG+y2#I_oKYMQHWu5bTdGcLb!@ z&{~AAEu{1ZEb|NG+{^seF#pSl_Y&!I^bFE}Bkm@$mscN?($h-Vs!M30gtb<}S!4+{ zOQavym@91@IpZWn-OfoSbMlC^7Tv|1+(6D0_PKAdg`cI(vn=Pcyo%4t9L@2xSW{<6 zTQAJKBDV90zyt&Z3r7VxEG(aEz>%16&5-;`R`EU~O(_6EA0xK~qLhQqu_(TT+YI^u?fjiMMo=Bf9F8D1FrOnB*Fzol=?=S2Z>9Z3wBJVkHqqCf zqUF~_A8$+pYRhSPDs`sPT7TAcf7Wk*>TpH6R!nP8h>kj*{_2OkQpPdvIL7VCbS>iJ ztkcJtVhHVAvXQ;aB}y-`gnvrpoLBoPbyg97$`tLH$UCIlg|D8Z{YJ)$q-H2Fn>w5+ zYf;qUs8Y4ZT5NUN-PA0kowL-rn|7v<-;@0D#P^-gAgy#>-3HIPkY4T$o;CkFpgqkL zUuIq0$(B5fb_$4diS(sfJZ-+f6jMn*%xiNBW7SeKRiw&?VqZB!T*Y!8$P{Bj_8>o< zNjs_eqU1_BF8@SOmWTWv($S>PlU^&d8b$gfY4)wk`^@u;w4j9KBCdjWW(msEep9rGot{l`zC;f0)r0>ab#4`OzE93wM2zlWGYt3;&pM6yw}=HS zVSD6hk!H(LKNYFQepyklY8V#m+&a$+9fJJm{}5{qg*Mp+6t)?aJ-f!06A-kTd4DWq4hH2)8setTnb3D-u2O>ecA{A{t>3W+(e2DxVr0=Jla;Zn$*MdIA)rhNJ6&fgcew&&Kah5F=3>hq_^IUZ1 zT+bwjwV>N=?>VORjPSL+#1q8T#6J+9Bd%auTERM6!B{Ioyl8uNdA73EwX)T$FUa0v?VjHcsQU3{|-OisP{{<&jQ=os8 zb9spP3}dYz{gj|SM>O?0q}f*W9@H633zara{d-BjMEWJxbuDxFQV?NBt7WW7)Tt#_ zG1hqUCy>)j98c^|et)4=`|3+3Kbiaz(l=20A?bM11-@smpvv2JWb`L_%V4>A4j<0--o6op! z2(49$j@~GUQPV1LO!75q4yNW_(lH!G{DyQL`EOBkIPLf3IAooSMb+iRos>p1CtVq9 zGVNC}tvSpO@0Mux-OhQmlP0L_7ER?cQ!J3U%5O+>{HAOm&GDY{jEql|ztNhV;&V*# zCP957>3EjIlR~SU`6{n4xBY~#Y~jqPiMi@WJ3W|llX15YGlUPXi=629Q0k#3p9tuW zQ>TP{j$zdKB2|YGcN4!uyo>2h5E@Szydj1=UF`r6StgE*zaRDZN=Z>a)~2 z&D>I|Jxk3^l$uQCN7PwC+)ONXxe)h$#$r2BE`)p+oUxqQRZ{11#+}H#^&{s|(r0Nu zUpU$=w(z%z|xuKcval2m)qayGGy zE$waUU!=80TDV;hBRwg@Q1badQ#FkxJWn_ZM`$YN5%`CaN7=1JDtbv-^&s{5sBcYb zR!=tHGkF636*pKtuU^(P&Ewdlh2uT%FZpPx@wUyJNq$TidT4){bkZwe#9#UDG{!xZVSAEl<=_ z@J8|?y;2Y8!}LaczkZTFO`oaH#e2k;>MQlNc)RyzeY?I}->)CmkL#!P^ZI3s_&gZ< z^>D;E5*;acSz?jW@1A>_Cp!;~N_l9MnFvgG_8T=N zEybAz%y8xaGo3}Frlj|CRsj1uYk{{pj{(!1gMjJIVZcl$QqSn;90~0290N>oHUm?g zlSWO+?B|>c?C(4WOmVgVGn})=OuA>3b8ho}WAAY;XrA!!{m#YB4~%MdE_*=el@HxN zYNB(^Ll5=q?_3W|b#4IO=G^?yL;d?Zw*ga}J0E(u@ge7)haR5zkaNEi|I&0?idIuq z66M_A5GVpV+7)e}-(TTqA2M)IlyfB7hN7qmA&M^hn=jkPmZr$x^;B0PzF&ZS2*%cO z-2Irncb-&2D92m6WsmQ$v-0u(WamP&L_=dc_-xJ*L4(@ja_dD%<^l5v4%S3zc z+7#Re_2n(TO!usvu5*oj92ybip9>xb_padi$Q=ra=Z*~S`vvz!!Tlh6?<@)Kixel% zD=R3+lL|S$F(tTP9+c0pkB!lG`8mf0_szk5=iq)*a6clrA8GF$mx9N`g8Sk29)5uJ zfqx!y?>H1ZE(`AQUV6!o6JI>wK6oDR6Fd$Hrt1inkI`i3(|N+KSIckLpL4xkzm7RU zJHd1vGwkDzJ)Vo%`Iu_k>3+uEcf*)L(l_#g_SXmH4%*jU6|P2CKhh4oZGD&XuyYfS zLfnhpZSKA9)1e`uF*e6F&$S9~eLmnigZC{<>k17$=}|(9JUu+Ap~F12o~F=AHs3QY zbW-TtFt2B}XJuH4XRGHhYs2G8b{%#{!sc98j_b6$pSwMD1!Hs#I}mmu{?C5edd|BriotJdF)Va0GrSS0ZO`UgjKGxale5tdyb5@tmowjt^*J*91 z6H=y1j~g}v2i&k7n0kX;0k_>C*HzjLhk@xg90z9HAXiN04d;PbH%M#Gj?jQP5guS} zL^v=nq6aWPA_iCxkq9h|kak=Ykp(P{C<2y5R02yQ0>H9}VZic;MqpLMIAC?eBw$U% zG+=DROki!qTwqGXLSP+p8%5+l-SKrA$$u=-{x|SV;fKNThaUnqgv(fOP`LPt`tbcg z{&O1Ndy!Z-1L=DrO0ah-A;_7g3{)y3{MaWe{UJAS^}Cf&1z&T(cVli;x+>k2?uu8D ze?nVdD}ybpUj7U=BpnTU!g0}(2VaaaV9&z0a5^hpVD$!kE2hKpO-c{st0&GY65o`G zQlga@oSj#RQ{t5bd}}9B>8&Io^<=!8;PVvp|HBmc-|_!P=ivK_vrv-~kbAYy5u~B^ zIrv#SaNlQFfQj$~Y9H@sR~5Cd)S#NY0cENtuNR%9Stt)RsYNMN)ZXwBz0YHVFYW{X z+Z%U;dT+!g{5EW-vB6*TK80;9wv*UaV>^LuCAQ<(mSa1HZ7Hl~C|c6KB($TXG)3!u zpf}o5QYxtZz0sbMQb1iwLO+ny57f@yyFn!*Mp_bDD)lz^-Wt@~)O$-%Z)5MxwqEZI zz0ubstyA>gkoMjN`l8}Uic0b(B_t)`2ygmN8k00TsX1vLj?UvKBB^^)&!i}vU1!Ca zGy}4ENeh#fC|aLDpB*SYrH`i%-v-bJ-*Z4dvXb(Wijwf%q&^Xf(|skhDzrAVK6I#* zXZA%!P29qEi5qmVn5u*%RVVqA1|?F&o!uWb4)z8>@y&$?H+f9ZbF zz0LiKd%yc1ZhWC3DFl)qxR<(r>RyY!c@e@%+Zq=J};(jc1+b zdCywUFFg33Owu$+e&^ZbdBL+DoL_rdJ--6=OGrjSveolj&jxVTdNz7q1oeWa4U#fQ ze(%`~PAe?E1nRf2@Eb^yAlU}aMp#-0>i4klJI@wK!Xer1dBwBK^C!XMH6uI8&e!=~`d!_rC&nYR}>9bLiv(WdQeojd~JG-Am)gVSqxGF}Upe28`=zwX54kA~n%TrYH=EkZ*kb)fe;lelmiJXXR1 zq5G^5`kHKC!lr3^5pOGG`})ihIw*7Y+K@O5Tjex(ym*wE>>-o$a`sx1Fr(L!#Jp*X zA%mBb{WaP2#Bsf*Cybsp4YD1OVc#J`ozLjAP(relft(Gl^Mv_*S|lXPdox%{rPVmc zp2$;1pNr#$GyV;bsM#C9$4IYNRG*`zsn_wB!O9u%-F2UpU9+$lF6UP$wKdtfH_M-S?|kz z93{i9lOT@?ogqm|I^FwN7LJe(&n{~$g=}K{9N$F9=EUv8u^mrKIIKkAlWcmJBg`2Vg3rac!$QM6VPRpN!#ago zVP=>Sc4JsX*bQOfVe}Z8`(eo!pXWnP;`0*EZEvC;*FngPQ6fLo* z(%tg|^=5Uj`k?wVtyTL7-`>i0{>-`B`M&e4E8q2iTk|Y5TzJ|43iD?3m*#p4y?3t} z_+D4NUZM}e*T{zGx9daoJ8)OvPWc|0?l=Bwyk;CQ{$?CB{%#yH{$U(8UN_)@(0<1# zTKwvG)D|dd?`p3>;LMLls{v(miolr?KLwnkAO~ZE_!03Vz>)XdNUTV3%HqpN2?zC< z;75zFuZy@K2dz6kfi;B^r4Ggf@jc0j4{~~d)0v!vAjb_3{0#JafuqJY;h68g0f*zL zgcXYBJLx-#G*^HJorL+|9QH{&o*(2)2WPJjJ|$s#kb`fE`L_GEgEKD384AuuAAD@W zP@5Bv>m_~>Xh~hpLXF229X~V3k+V?amtx}^fWVcrP~(?i8xhozvryy3H&q8Yau%={ zpJi{L6G6>XdinlTr53Clp8QvV>vremS~kX$B;_A(4=l+J2eoccr6ZE zd2}P<^ziM?JSgq!TAU4;TOg}|Oo`v>+b}TgnoJ$o1Ck`g8FM0jrEf{*3Ljd`wRqDr zB_HszF}r-z<7Zd2Uz4d7D66Ooh_}u+GJafUlW*iTndU30*a_( I(wn_2BExlWc) zu@Rj;9TJk*C@zAECb}MSC-L?#ATd9rU zZB!i5dwug#8{yk&F%|tUa_5NN6xWp9@OPv*|Bk~<9iWJQlI^En#FeV*q5 z+rjS&esbKI+@kCFYZdj@cJRB#9T)%_R(L+SxGHWvspwJd-nPb=>-49AzZd+I-laL0 zujAwVZ`}y~Uhm8Spo4a_xXDA|BX3=Vo!T7 zw)cDsGeGS|1)g1=93+y48x=Ub887&vmlaAwwaBexo=dTaL9(Lcc{Z~ByV_B)RF2vY9o$D@o2PquQ~=t z@&-pmZBp-3$Khx^kDAr{)d@IyfJYNC#>BOxKE$J`_=@s096idT>G+y*3yvP+kzcKs zcT=eiJQ}XvrH;VS-8`DCKCHrvtB>GFi(ZPmDjz5xDjz8yEB{h1DW57=RSos#R>M$6 zX(*X2H6LYAj5F8O5Y(ZeKB;~O?+N>^x8#fpc#*Ic-qnpv)xXI{Y%*I{mxyC$m zp}ELhY~o&0^b%#L+DA=Bt@TG4m!o8dD(@(#l=o1JDJp9Zd1JjHf2=Vz5BX$mA+M|_ z)K8^ag?gz~>r|GA+Mu#tj4nnv?uy@HL>jjmQOHj;aukC+#UfWeBi=|r-g+T-iO64X zgN4ZBij;|QM?f8Rhx;su2=$)70sBy4XqYqTir*)o;?1^?rE9_yL|9UeGVKN_B!@C_B-Bn&2-Ijeb1fa&T|*Ii`*sdGUG+#H^y&`myF*jTGUEq zqS{BV|Gz8$ubuh)AJog&%%;>{tCcHpE92J2wZ(0Y+a9+&Zhze2xZ`oB3z`ew!t#f)H~Zz<*jYkh5)4Q%)Ajvj~Ez~T5t-*MmR=!wx&qNhjC zjGhxcKYCI0(&!b@tE1ONw?%J?-Wt6ldUy1`=!4NmqK`+Piar~CA^LKR5)%?*#DvFm zkBN-&#w5n{i%E;gi7ASyh^dVk6f-PlWXzbD=9oz_Q)60UX2r~nSrD^0W?9V2m^Cr$ zV>ZNWj@cHoGiHzPe9ZosLor8VPQ;v!ITv#==1Qy<>yEWzBVv2RM#cJKlVVe1Gh*{% zOJXZy{jo!0hsQR?HpNbeog6zYc1C=2?CjWiu?u6D#4e9r6}vXJHFjg{me}pFyJGjo z9*8|0do1>3?3vi}v6p<8yRD6FkJr2ougBXt-qZDvx2w0OH^!UbP4=dGv%Ce~GH7C=9?_K0w>RsVo?Oo?>^KSBP_3rTQ_U`i@^d9jZ_nz{e z^hvE~EjTyv*+(0twe z&~o6(SH4wc4YuyGriEP&`^>o0m|;F-uCn5dT=T`SV_}!fXU%Qa*s$ZqtgttY#m3|K zNNSkb#SAs)nG4MM*3H(f)-7R2!`=!zfhTgO!cK?1ZFs}Zgq;mL7j{1ELfHFZ7sLJ) z_DR^MVOPSg%FBrG33}ad7)~R^a2uWQ1TWMGGYrEtETglLW8@k6e4bZi6dNT*nNe;G zG%AcrV~|m0R2wx$EnZMxZ`4_eF~qpt7;4;M3^RrscNtB_2;**Jq;Zci3eN)XHO3f| zjQfoV#skJg<3Zz5W3usx@vt$~_?9uv7-x((9y4YdPZ&=c-!YytzH5BXm~T97EHJ)r zEHr*#EHZv*{K!~h{20#@pD~sh&l<~(pBO(iRvOP4tBju+tBp0rFO0Ru^LX0$OJlw9 zE92KjoACmkJpSI;X1r`{H+C31jXxQ?j8}}^#-EKn#$Mwu#t~z`@rH5Kc+>dUcneP` z-!@Jd?-(bIca2lV8RLE9tnq9W@o4xQ1G|5ah`v+;a1*UU5X%>uK~EHaDD60_7SGt12yv%(x`R+`mjm04@n znSQh0Y%qtIx0^%FJIrC`NOO>Rr#alb%N$|eZQf%xnxo9o=Dp_qW|MiJInEr9C#Vz5 ziROdmBy+O)usOwi#GGn=%ba0OGpC!6n={QP%qPuR<{a}o=4|u3=J(8}&F`B(Fn?(N z$o#Rn)O^NVW-d2>Vy-ZMYOXY&Gk<2THh*reF@IsMHJ>-vnZGhy&0m`v%#G%6%uVKR z&CTXZ<`(mJczXML^JR0p`3Lh)<}ULUbGP|tbC0>#{HwX&e9e5-{ENBIJYfFK{JVMB zJY@dEJYv3K9yQ-IkC|_o$IZ9R6Xr?tUGtRrPxG|-o_WT6-~7NlXP!4dGB21Pn-|T0 znU~B@%**Dd<`qjf+s)4`)w6@J1tJE2x1O+OSx;KC@tpiS)*NfD^<8V8^|bYUYoYZ$YrgdZYmxOsYl-!&g}LAekM^gq zzlR+P`$yQ}|CQ%?J?u!>oA5Tr;e#&OUdVXdm}M-6w+W@cc?kYyK0MAU^I7F-iJ5(0RHGhc%+ZulRo~EPg1Y*N-p@N zQ23<|uha>C=|A<|asSIvRfn&$JzXh%9X{VgZ}$aXht&;(zSi3{Uh{Wj;n$k4^Ldlt z^B#iFn*z^g`@Qk-gVT*~U-Nyl;r-^o|ILL5ocCY(z!gC+`17D2{P|aS!WWDU7;$aH zm}?V8U7IoP+G70fI^Xz*ukenq!awfAXlDPv-5q`%_lJ)Jz2?9B%=hG)y3TWU_|1>6 z^PHc2rSJTA&lz!@@4Ok_Q|^S{67-<4@U-4*el&P*{7e2+ylLA1!k-TQFFfj)>pbfH zU+Gh)nUBJ!e%ov@AA^6r-nY(%cYW%bcU=hox(FV2F?{Tj|H{kao<2Qo+c)^y9p)du z;BQ}r#}!X&dtKY#inl%d75?^}|B=VN8uYpNOV@QiH|(0%?Q+fU-bl~e&FcO?@VlvB z>v;>TLipYidf$rw%Kwgl{~bvWJc>S8{B4IHzTcVvKRglsc#`$df8~Y0V=aIee(G!d z@M3u4A6Y-PmRiqP%WR)~-$7?6ee(ZiBnUr#ZTxtQBgZd~8#!tW6VLs{aU({MLErtg zK3g3se!F9IDgOJLMwp%-gdMeK$(UPyjCti%`0!7^;HSUnr90-~ws-EBjsK^)E@ryd z?gBh0BT>6Y3HBggH;VoGF>HrlFUO}N>DA?l_Vr^}`1OF%G0y$UQLZuQYsa~bT7=e> zzJH7{_N#jbEh8xZV!!a^KHqHGszr4 z`lf&H%{t}*U!Da7doRab|K5Mex(fDS|8M6D|HJu(W2ZToGmAUFG^_aH%;VqZ9b-82 z_a6BOUZ2N>*rI~diQUbab4f{cg(l0_53c^wVwa0 zW@i8UdD)jo2f6?C*y7)3bh(y2qOfOk-#oYb=K0 z|ij^_u)iqCtyuej^G5A&q=&_4{uOrXObiND!lp2ax1 zi}#Pv zXF8gC(kPc>?c2t~vUe?Z;S3c`|Dz$#LN(_jCe}$fr#K&Uev8Ksn|P|EXdw&CcV+MD zVRe$dJHfgiIT(rU9^?W~RW1B`*E18_6WAWdHV1l+(BFo>?(v0{E4n?~Qk+e@oAx)& zX`0`(NO1@@Ep5X3fTm?lE1T9dtw#!TTu%X4yMB%{e+6fab$uZ&Y@s;&9gz54^@=9G z0QPn5W-Z6KSJ50+=%>KWM9YJ9@{Z0#%Y(dW;n*Uub;Xv3Ed%9*XKqSJ=*yvRhrZ_t z^(1>Lajb?yTXDD_bWf6dI6}K>U3IQ$7D^UpGdA?;&}Tg;=}DNS9+wvRoHqa{uMAA4^Z-By-#4c~+JUP+e2%*^R9 zIt&goGc$7>W@ct)W@cvSFm)L1FoVBcCot)me%|%0kABT)&5)DbPU)=t2F`pWt27I#azW!=i=Z+P7E2(N&vf?pNYNZt9+^kwS$H;IJxe!B*7Pdg-^NYzH2H+D3N?MzbVbwlO$*^G zX8GJr`0MO&FT2;=o1Vm;WS*4XLf)cY^FDYd;B|%hjSTMHV|YxzmwuPci^&Vh(f57r z11kElA1uceu5x{@-;L--cBA_7X|x=+o0k-&L@ls19l_?p;cRhlY%i)Kyh*(&E&NsC z0{#bgu6h2S+|KmZ+UeeMA9|E0m8S@PPP@C^+wNnJ?MdT-8{AUndWE@OWv*9x&fq=d zp2uFii}YWF_l(}D9k_YjW_b7DBf#tIckjEQ9ID zK6antClr2{i=Rt{-tZ~f>-v=IJw_=4ZjWwBBlbbb# zSyR$gkv|&m3%xrY^X>%V*+MuaxD`w*KHc=VyjiQ7HHleMm^F=AGnzF!tt_p1e1}%- z3HrC?%$mWh)y66tNFkPprDB*esCZ-n!Fx961kdIcQko8Ua(gZGZ%uiD&dXLYi=TfME7 z)^Y2!b>6zj(zEI!ROMAo>`nGD`<)Zd`xlhnzFjV%gORkkEog|4w!)Cq$ikv^B24e6`|NJ;H&v7>ByXFzFMH3s%Lf{yS2kSW1#Tl z^v#AI0lwa!iGKT!Jx@C@rui(#`|VkC5}406ed>aAC40TS$L;Qpa;Lis-IeZf_l5h} zBRmm2kvx$-(LFIxM;6Bu*OSqc)sxMW&r{Y@!Bf>!-BZ(3*VD|?!qeK*-qX?3+0)Gv z;_2-f=b7l4o01sGxl+2dP+Uh_0er z=^Hk)Q`&v(D>$X}?kV@Qd&WKMo^#K;f4VQ-SMF=~jr-Pp=e~D8xF6lG?l%w0hdqIwh@PmPn4b8a z1fEQu?4JCd0-l1NLY~5?S^w2>1pTb;OXj_?#<$DA*69WdqCM{SCIl;1YNRm%ux?1U{jg7lWh5W*+z-vcUV01CED$tN^?YaFf6> z(2xyagV51?7^Yy2W#idIK9Nu6Q~7j$g({n1aJVwwi3bL6W2=Xz2UuK zrGwh>)=Cd$^pjP@`^D#2#eGS9NvtZkj#xjggLW}9SHsQKIOb|xb2T1wHC}TyK65n@ z%+*9PR};ltO*C^gvCY-QHCGegTunlAHHppDq%)^PdUHzrVXiEfxw72m$|{&Et75LK zp*ceu;fm_v`ikMiC}UN#YT^2tTP>`XRx7+~9dTlHwz^nztV_7EQ2dRbn`@)@iQc7e zRye{2TvWzAg?EXrimr(qQYe-1~x!!}^K=&3NCiN%&WqEyP-c?Lw41 z?kj=EMS;7yuO2WW_ca2FrpI(Wjew@PuRW*V-jSEW+6l;<`v!7)oxvQ=B;Qaz4eM|| z1M6%KCjq_-A2+^x9qUlcsHN}zg>^Q6jdiW?VBH|-eb|KQvD~*sM8>*JM8moR)04SK z>f41irEfQ$*(cIr-7o&YdO&2vdPvYaa9C8vdPLyU?>j20Vm&6RVLdLYV?7~iU_B{n zV?8a(V!dV_ze3i+#d6;_Jo5w2^9!iu^$v(oeP z9D079L(eZa&!cY6Jf`QDo9EY?=hvI(QCElO;T^>a|EO8%=XaUsQFn*O)4d$)Jg>mI z+>7gU)_d_@IlH{LT6`COS$x;Qy4;Ju4!(=eExzkDEB@Aa2A^Sk*N=6cHvsE&FDirZ zGk9dn03==Z%DQI;wKG_r5Oy)`z}?SReZmVSVaLjPr-ApSFD2I3zEoJ> z`cmUlYWs51r`LDV5~!yfi+*JSRh3hmnYf>i8p;K@Ux7->mAK!4ddiKs-;JuuJ-9!D z+RCH2KZ^=W)B>UY@=xaq?r%BQoWCsLy#YVy9o*l7v#lEL|01-A`#W$Mq6dNZ6TF5^ zaQ^|0LwHQR;qV={!2NBw4r}24F1&}eaDN|X*h}0$#3}X`_m6Rop}x@j6ernNOZbw& zk%*oL-e+)<*1>%!3u1L~{~QiQ)Tnu1!cU1BB=2iD6rQ7 z$(V{^Q|1xI5`+{0_iTH&6<-PjuUl9gu(aIfHVY=e87s*-T; zu*xhA?oXkgc}&Jl_mp!Bf3V>br1zfnoMi&$klqK@YxrZ|;L#)NJtj(j#G@zH zXSig);87T&Qyx|ik6yqjnh#!9i4)fW7HNP-ZxF8vv4)n!$s3!{uO$O|{Nxu{G(_n8 z)&oT7N7iGOl2w5lwJYnz`m*J01KY~>QD#9Z5IfDzvGeRAyTq=tYwQNQ#qP0p>^=L! zzOrvzb3gLlC_E14%H~IpmP)(|ufyx{2D}k(!CUb*d=wwe$MCUy3*U;~EW7z$et;k0 z$N4Fa>2&WU_!ndl^Xie6%b7%j$%@nQ<)H>6jHwPL;4 zAU283V!PNO4vQnm1|Et>;xV!Tsu_!pUM&e_5}5-1TGGfoCOar4%bNawP+BkRis zlqZpzAUntb=nOVlPLXrvJh=$5`H(y;FUjljmb@z;$mjC4d@J9}FY=qRR16hUC00pQ za+OM@Q@K@;Dxu1%N~)%+r5dP4s-qg92C6~m2fbXaP;1qCbwC|gC)H`qz&F@BP)E=a zbz+@VC)X)-DxF%V(?xV?T~SxojdeTSNq5oRbPwH6570C9LcK^Y*6a0ly<1;i*vah_b^*I2xDxO)c1632UCnN9_p*E2gYEHPp(fi??dkSRdzL-N z-eB*w588+A3-(3(l6~2}XWzG9+OO=_j^lV7pW}BTJ5iiCPEIG-DdjYFx;uTGz7F^$ zeBS=4LYmN3@&6C;RrJ3htbfPb6$x|wJKAOo*ly&-55UL71{;%`7vm*)&tDjP^-mPe zU;PcOB|`rE9kCS=MMbbEh7Lg`1o0@PMH%eZ>Bz1R{tsx*0iMf=y!&5>?t;9c9Mws6 z`Gxf<-#qdY@l)mr_9>PgqwgbAkA*ChxUSZA_wPuMxIrhy?|6`SF>qAG_Wa#SyKhH!zY`h!ZZI5&Q3-w&72(HG1%49M;HSX~#PP=U z#`DGp=aI~t+?&Fi@;|Z=?K>6FgV;CVvqDx8Mda&dr;6R?CV5i?Z+EE%wJ3YLnQx&WM%KfK^S9M$bv z2i6hOWV^se-<$Pg8`&v#mi@^tuq*5)yN~`*ird`bF8A^P%$|*iPEk>LG#;JD;4#rN zDmIUYzERnDc3zXWUWqdhb!B-;tU⋙N7Q=0fp6rS_-4L?@8rAq9=?z7 z=Lh*AewZIc@2C^}BtMG|Qs?NASWBKMEaMp;ylmKQZm z&1gN*Kr|LjMQ1TlOcImDRP>aZA!dr%Vvd-L?o#u`0Gog z#8$BrpQ1oh^%}+0yvC3*Wdd}ZN-R^$Ak@QFL;xPM0&~EIC`wkqhNwxkNseUuC!oRB=^26<;M(iBuBw^-ZZ#tMn?f zDxeCgLaMMTqKc|uRZJCEB~>X^T9r}dRC!fFRaBK#6;)MLQ#DjW)j{=F^T3}hREyD7 zXQ^7I)~I#pqP#_IQ@hn3wO8#^`_(~pNF7$k)CqM)TiVgC_UkA*s*a|k>liw=j;G`6 z1UjKksnhE}bYWdo2kYXxq%Ny#>Ds!kuBYqk2D+hcq?_obx|wdSTj-X$wQi%)0acci@ zR2vPZQJe4BuI;hCw%^Wa=R((>Ane1uc0N15UC=IM7qg4oCG1kzndR*Yb|t&2-OlcT zJZ*qI&>lpQ+a6{QN8g~4_E>uyBKIVFiapJqVb8T!+3W1>=q9w&-fi!*_uB{TBldaw zx_!gGY2UK%*mvz04rp43J80zWfY5bp$8{n((VXZ`3@4@&+sW?aKo(fYDeM$+iaN!d z;!X*tq*K}{QFr3j3{L_#A&d|3Pt|>Ge1rM#1`~xuh(LAf9&=VY%-h2=Htv>$X2uMYzN!P z4zk;rsx7h4ogaIg_IGlgf~Vw#c@bWe2lL{*EU(5p@Q%C_@62cZ*x@VrDzn2kU?-pD zw|OWJyDV-Z-&e@^ZLp3m_{soDyPNiQY?)o= zK-|a$_Oi4rW7x~zcX=&Y85Ke$aI0YWz z448y-^1Qq#Z_0b}KKA?*`Bc7YLfww6Ei0S0~g-bTaH}+Rr7hpG#p! zm(%6VuC8PDHSOqT*v+(;+hZSh#y$qeO8~-7xis@N8i==^aK4+hw10~wf?9->#zEo4mUgAu_M`$ z?I?CsJDMHGj%z3Q+paHWH@92ZE$vo_!fotsKlb@pv!mzRE3lt;+NbO@*u^)^4t{Ds zw_n(A?GN^6`^%5r{CywO4o>1^`L%zGVds{0$~je>s!lbh*00^$di(cazcZh>aB&D{>&Xs5diD#;$`B>SM09B{9=SD_?bcW<~ip_}{#<>U^u zlY3B49zZ{N1O?>@G?Zsy2B*ni5LvDf_9 z{kQyg{b4v~|8>eToU=O6#VH#Q7#J7tb{M7vtupyh;E zJ*=KqFK9Y_tiIM5YpgZZT4AlRwpn`&U;3RNMeQgU(FkBfqoKNv>I6~UiL?HDC14Tc z@4?9BOCT?%8oU~)!b1h5sj#Dpx+bWxY)#dZK=D0TA2#<_ELg;ru%&DnTftVcRfr60 z**eUFKf^Aw>l7W>UG{)IWRKWm_JloU&oCY3IeWogvX3BF7&?dw#19{j%j27867zeTY2va{?WyUK1d1QDwz zVpeb2NA{Kd5WA+z`EtRJ$hAtYmTTl%xlXQ^8{}rWRc@2pz+ETclR14Qi{}u6C%MYL_}? zB3`I^u3o5D>a}{K-m3TNgZikxsc_A;&`SGsBpn%9U0R(KRb5S2*EMuaT}M;o8>vU>(TIIxe~W&T^=v&~FVV~Na=j99@DL*2 z_jq?+|EVt^>RtL3_wJj>_gFvCPxUh$YGU9^{R(mLt$vSK7;gK1i-6JXn8-V0n@AYX zj*n=V&`xA0wv*UN?NoMZJB^*zPG_gLGuYYf!gdk6D7tA@wwwJeO18B-*c}loyW1hi zTl?AbiN~^+AX=`p*Vt?AjnIBK+gt3d_94X1(}fomhyVNu6X)a%90Nom5VG^wZ4XWO6biu4YAEoEu6>9%v=`pq3Q) z6HB7R=? zBt#@kZX#eqPh)QZZ(;8q?_Tdd?|$zA??LY&?_uu|?@{kD?{V+N-=g2&BOg`v{fv7* ztMO*T4fX#<%&Ttd)c!rP!;z0M@<+-I@EvtKaO+5O+D3o|3d#F|5E=> z|1LzgEmC02zyyH_0}}@(M@&nLc$PH~ z3WYTi-7ABjUe>poLP4b1k`T^;HRx)&9k=AxVe2e9Il@I?WkOWwV`XFg(cK_BRMkON zPAIEGtXxo6hgrGV2sYXZg33D9$`7S=yj6fr#3a6g$SM|Ch1ept$SRLWvdpT0NV3|h zj5xC1s?IjC%~mZ$lIxh1{1>}x)kQRUY&AqAd1W$!FnG~>gqeFEE zYY$>T7wZ_}Ko9FUqQExml-MN>TBi{OPFj}`0d8B@>3p`H%d`@#xXdbZS|4R@ncMnC z=d=}0=QM=t_Od+_vXkrvAu>evhn;Gv+`uB?1m4HuDW)9E)buDn%ZQUU63a&CEPB?b zQE6Ci^@qyDf>aijh2_Uto1GO0>`NT6;8z<{aOvZYn!qI!3Ysq@yBu&fu z>I^zN>xXkRCmV*dG(Q`TbF>f}jgzz#8-vrd45r&6$6(|1Ks^!@Lr3e0Y^I*9@hQOh zIh!rS89JXW!s&U4E!Ic$Ikp6+=bvmnou6z2&P5;FWFy&So9&2pbhd@gMz$MgV-mL4 zu5Pzx$LzLtTXqZQT?qTjo@Xy$k8#c|VbAESW1%?f*0Sd~o3ujs^&YZYTG7b-)laBjvqGjcgoeEAJ9+OTgo($-}D^KC{aC-2xbW-tj zIIH&ZKX6(d=NX*a&TXC%9H8KtJPkb!d1g;zPh)&GIjIxmmjOy7=M(KNFF3nez=!pL z-tt;QajRo?T3z6{sd#oeZVe1$*ATh-BC9db-CC;&@ZEOk<9n>V$l(uz$!kYkp4AVS zFU%SMP2z<$7&`eYYY0?{H+a=|P|Jsb9sXjChLRuG8Uq}d!de4C^n~XD+-Kw7MPRx!s0-SSD@5jpF^j)iCY=k?ZBe< z(PzFQdy89r_JMt34S*UIYYbk|Wi5$wU~RxG2D0|VF0u~b7h|(dgeF-JaEobJZ=48u z*)U#^mx8;a3@^v#@Cv*lo6jrr%IH2pE2eYT1>0MeYuPQoKHnVlh@2s;bhxtaY5oVklV2RFHgy(WH=y#cRrn7t#0 z5=i$1Kgm855@lb(QeFY-#fbww1`ZSam)r&#_27K5c_MsZBJy-#K4SB1U_TP^>>{a1 z%JTr1rs8?QhGgUgz-MOR#c&1{;3dF^l;>5@5w8w!NL&YRBN~auyghi$ro1E0Bd{)D zIJ@vZbRO}3qK_EP2Z99|#m9mP8N``ZVRCm=~2>lsc;RjG6Zqg= zih)nM1cL6SeyHN;$NITSreEr}DlIVcdzBHm`J>7N9_h2ntiRfp$^tG)svKaCwaN_! z*{6cQ9|x)e6ggBOM2@7Y1Q2w7RTV693DuSuWYvv0WEBDyxw7gBCb_EWjkwZI^+81G zp{4*m4^s2MDUVc3h)q_jz$Z^oYrrW_Qft90Pf_c@L`_rc!7tBH8^AFyP#b}^m#D3X zK O%v4HxT!3wGW*00dXgSjqo@~zGa3xkdS_g~M*j$BCGhejcOdFq=eQ%m4lQ*jf*o4!&HxX;&YcM^ev>;7 zT>KVyA=sjA?jmsVyWGX#jrP0Cz#AQOSAjb^;;u%w|5NT-aP?>1P2lU#yPLr$U2wO6 zx4-3X_22g2cK3o;y6f%(kN?QsPh7rx7%2ROdlXF57xx6=arZQsCf7YrOp|++xF+`o z@OfnS7MT6$?rk9Txb7XG^_1?zz|?`M-Iu`YY28=E@w>0V@@I435*y`y3d|Fj$A!xV zar{3qM`5clFh^0`l0g|R1}t9!-n$fFoT{KheSP@u(phbRNOBoOQpzBb(gu-~H7KN< z;h4%B6jH^YkZL9ys9~~!#s-BnHz=f)K_RUT3Tb0dNLzzKIst`5w>krb#0M5h2NW{a zppeN1g-ihoDQ8Uw3TbVv1ODg%L_!(H36o)*G#SQe!XMTd!XMUolRbo)?BS)!9^RVl z;RE3h>yyDBaI@j&0C}7Ng1L;F$6yYxK^s2g7O#L~2yMg#+W5%g0c}W@0Q{7XB?YR8 z%aT#H!ID$9!BP-U#WI4a%F8kvgptc2j3NeM6f+2;1Q13IRuUXmZC2LcjPeF&R4_QB zqQM!J49=*Ie5E_90dA`|YXoL%AZupOMr(sM+8MOb9%y3;@Xac|iuDEN*v|R^Rh(h{ zDVt#fD4Sse30trsge}-mV2d|w1TvTpz&$`1Y(60jwwe$I+X94Q+Yo*L(bs+fe#k&0hCwqL4*%5V;A^<&nA4pR}ntIRCnM5%x(id;G2OD z9`UVEYr^<;LJ52ip#;8{Py*ivWZ>ufk&Q*=XNmFRmw*OR@GF#)@vD@R@oUJ*vcp%L zN9E^t!F^Td1_$s*gaCLb5I}qWmJk3Yd?6b{!)_n|5LQ3{Fcbg*2oDgzT+A0|U!^Ln^BqtmoQc^xAQV|XiX$c32bjakwMFzqF zqB0=>QJpw3QA5Ynu|zE$N5>U)2nUFIgabr=t$X~=tS9`=uFw3=mJboQ*;A+R!8(A93XlV4iJ5S1GTaHJ}9xNwN%$8FqX3J@mkILzkkII=8v*j$xN97!f*>Wz$Y`K71VwJ8C~~WaP^0duNa)-Zsv?7zvsDymQhpT!x>S4>6WUaA6$|=Q zE)|=4bgB5zr)sH$;OUyGB;e}Wsif!;FjA$2N;OBN0#i3%Wgvx0Wh9MCWrj+%S7jxo zN@asqbwcHUUUg38gl6?nXoGmK*5Tw3PQa~r-GqiWmLtW zVr5aqp=4!OC7@>IQ63g{h*R&`aDxII+~dRA9eh5DnY8ql+Psanvq`l{MM6;o9m zXj`jPU2uK7R3m6xS5;%^TK7~7aD5L|8}NP4RNH{CfH2h&jNcd4DIh!`Ty=xGB~*7P zTpksIH^!%WqvuEj)dxCPB-IyMS4`CpdRJ!EA4*qlH56)BK{bpNFEt#RS9vuOx>t2I z3ffmqH5&R?3pEBBSZ6hsbTBmz-Iw~Q3Fy5vL`|fAA8IOa&uBFrir82+gK&?U>0j+% zrxp-rsFncV>{ZLa9qv~v2;Zocgm2VpaEQ0m20}AxJE0l118m|WwG%idOzrW1^M6x& z!74Jf4|zvAmoND*U-JJiUxJD2gurwY!lr6B72a6A|{ZlN0u`QxNvDQxf*FX~tq6 zJ2hc1I~`#!J3V19J2PP~J1b!?J11c;J2zo3JBYBCorkcOotLneU5K!kU6`F;DgiVxH_Z#5~#ShzEidZ#pQQP2g--BCCc-zn6l`i2{+vs9-j2EgNI48?$0L?0b)fir=NLF+Bp`uwW z;D3#4wKR%YdnjTVtsdZ?f~;WAJx&CO?sKNi1adR3F&3lO6X<%tkrN#ErQlY3il?%QEfFG)lO(+8?8O0h*^6{ z5wrG@A_gT62+ulXl87UP2K|WW9mUDvk9P+*NMt6WmpKV8W`&YrzK9mbYaJUaNjg zldFnFGn!Wna#yjKV4mi)*zi?tW{Hg2mD#9W*^Szj!>C=kjM|mQs9pJt+EoB**F{zo z{;L-(7|z}gV8YX6#S>XO>U8bIyJ%$kFj z%FSB9k5z!RBh8C-Ce4fWhvrp}je^Iw0UIqEi^gmWDPL?X99bRNI8wgYWW!EPg(Is! zn*p!?WHyt${%j7p{nqA#V!C(Vm(fabN6ZH8O;5I8d^ zUu-AwRcs&eRqO)#)PG@@;Kt(Y5xiJl_KZ|8_J&k2_K{RD_Jw#X%%X=1#==PjLpudH z05~TV3*nfM*BeHJ#@~oxvO)M2Z;CPn8DX)s}?wC5pZ;(>PZxTZWPVS64gGmfJY#BppqZCLK+b1(&u&R3TrQs0wG=I`DkZ z)I?pXu@Ft*v^Xr9!K-#kw18jj5?H<4VAnbtb}a<#+G{X#Z^5w*G#uL?!?6v9L(LLH zNM94fNM94c24LDoL0|KW@$jxi7ZVKAHVsT$da)SZvL&wG)TQvnI~Kwblb19&|Txk#sk4opd*Ghjce_k90S2pL93z zfOI$UkaRb2dvoo%;xT+~^TiY5@WfM6-$WRxZ{j65yshF5ImN|CRDSFR{|6-w{2%do z7L>RnXd7_~tR5#lPHNKQq)mF93@{8|AbfGpWCYUVWMnXWUu9Id;cOY7R5_Uf6(W&j zN>buvD){44$qewtWsuq7i_0o=kt!$ik}4IK#d#ZU?^^zx{%NN8}-dEur@E=QB2PL3fQDaQdp7Lb$RtSc-hLxT&JQwUAU zse~rwLPC&o5g|yqj8LOoPN-3?25PJ!*P?5FExC@cqg+qeQEmWgoGG`U*Zmy1gRrCA zMc7g9ChRDI9DRG`e!`BDJmx3lA;OOGFkwe|gs`JLPS{bNAnYhl5_XiQ2|LO&gdODt z!jAGHVMlq1u%o<8*il{~>?p4TJ64r9&;hc#{EHBzyiEvF-X#Pn9}G7jCt*sJi!i0i4NQ4W1;ImjQ{^W#sR|I9R7D6) zs$fEsssy1)RT^kgsq!G-T~&!NrK(D3Qq>?dscI3LRJ93Bss@JhYehIxwI&>?+7gab zod`#&E~r;J440J8%46k&SE?|OV^QEns%I@>P-AH*mu3G}?^xw;6_2&8y1%*TTLV%4 zZ=Cc)4Z>UmWVsC}@~**;VgIY%BR{?N5r6AH@_T)3LY9anW=U95mW(9_+WbDZ=er+2 z6T_UP|HY)9?|%K_K&B;WiqgN&>p^8^7ogA(sGn4;wF{NDd!d0IWQW*cII&I<)->KL zViMu30*V9>G;VyV=5pak@|lXw7^YG)E^0ONLgOs-tFDV`bxZMbq;>M@gdllS=$L(= zT#h8fh$`MmP%fuHx17eOW75&z{rL;{!XF;}?-idv>%o4n_k^bk7;``2NubHIRO2aJ z5kPnmom(QJcS~e+Z;6KfEir)}ff-HRCv|f9-&Tp$!tBsGK#%ZDLA~rN`T->lFm+-> z#V|3Pv`i?O6Mj^PPDkC?tbeQzUG-aa=)J#H9)J1KNk#w}YExI{MeS%r%t?(6G?*AJ z{p7%aR8>|M9kGD)Ow}m-P5)Lm+7?x#gXNGPHDx2^C^=e=kz?gJ;Jit|cT`6@<5wjq zdG{AW$vpO}Z_Ro6r@R1F^NPGGuR+(mBOd|fJ(bVk&nItwTvKOCRi;0E`2~Jdng;)^ z#AflmZ;t%!rapBakO(>QZ~UlCeepjy@~yvBszz`k0*gfbRj>M6-y&11`n^uI+;5d` zt$x&}hWw~aJ?7j&HM?^4&-ym1RK;QO{@*hDOgN#lz|9&PCv-9RTIpm?WtBz_o(67` z2B>Ii1W!pG<0%O?o|5XuQ&Pj^^tDV*U)!9xb&S8Yo^h5mG0u`^#^2h)9{*r#iU((Lq)fSvpC4_=OMq3Z!JKDdkSkI&fN^i z&NIPxveEcXHW}Z^X5{_3;fKuw_sI@)0jUgsNj3C3IZM@0)&+D9Yi(U(?UC!>H@W^J zlj}b*x&CwV##%3pH|34-ro3e%(NXCGn}YnGlgk~RS@OvSfCE?suL=2NBO9M=G@R7i z(4%S>&gobA(}bLX8)BS%QpnHV;mC!SaJAuvuZ8?=CB1d@V(db2b=N`LS7XbScAH;QTMgS{lDgE8}-*1HVgE))o<=Hfv}6E}f0v zrHk>qbTxjLZpQD@3w%R+_;b5}Y3OH|hOx%)GS2v2CK|uXB-C^*VN=l$UO9^;JJ%kS`K z>;P4RvD5J1eqiU|j}dUXD7a*CgRQ2;8`l5MxI1 zXvTpX9n8cG9ve|+K97&y150>Pi z<6}@)){~D%Wm!Kyfg%{6B1eFen1+}&lP{#c2Yi{~BvycvSjJb&Rp2DngOk|7x1qjl zKO8oQWBd@sFMibca*iW@-QcH_3maV*stN~v7iz$%L*8DG@%9G7+0#r!pvnso1(g>aL{zXE zokeu2YZEc4@8xygtc{*C#n@-R8ozgU$^iJ9&OZPU15}9^>`Ni+H(9 z$2>E^n*Sgu=w!&rfy5N*XubcN_3Rv8b_YVrb$HRun~ORO_Y#(MaH zhKdbTV<)zv#%`?GP8A|zhha5#!Y8yq>^2UegXkHtO&mwO-YL$4$v7#_>GS%6xQbf4 zOX3E_aB)-L*Z0M3@EM`v4)}~$;-P*6KI5_BGoB)rTjCj_xfG#>$p}MK_lf7^6c(@H z8Hy<0+R?#hyaS&RTfDbXfhRtp0xu~z1GtF56QB!)j0ip>7)${AP{^3XYRK5cYQRMV zRs${~uo^NUJVwpnC2DK8lNrHnbd{M=kJkg9A@Umqp;N^mnU@$2nGanm#>pbYaLA&> zaL8a{IAk%f8uMgvdjXtDC5?{RcJzA9(N(l`E)ghFnQqGvsP?&8RHbI@O(;auezU>&dO?meE%3L|2{mau+HD zJImcpH+Z4;kbhb3HD0I##tU_jlsI|FFeS$gQ*zRHp-vg5rUt6X=t;RpP4n&Z?Nf`7f918J(CRy5a7+nDp13$q{c29(e~%<-$FV`5TYGo1(% z`8w(hn8DXw=S2VVo;o+pi_$@uYB5&lCC{}kL!N707W4O(>I#_rv|cv|*ch-=H^o%F z3%VU( z;pv6wLLX5t!9=_mdMTz(rqio2YblpLK$GqCaZFY!q|f?``pfBam}XZ^-@pvJ8u}jQ zQa0E3F|nYNevKJ*z4TX1rt51fnoft#heQ2CZ3lDdM%XU;+mEq5m`*pz_L4)~jzFA> z9mBr_YF#Y<8vhzQj(;80y0}KIi)Yljghr=JW%V4y*j7FQwiViO??Cj*GvU3EYzl)s{Jc?uIMvoVtoyVwhd5tQU zA3RDdy8!iju?vAsNo*HE&zGcjad@hd+a-)nSJLQorO*Q=gIyZ@N+!Du)py!ujZRk{ zXnwijS1jQ6JgEB2hg)3mEWuVh;P?_&B6K1sjhfH0U{#W$k5o-~!)xP~0lZ3G^e>?5 z&`e-eIzX-K`m2hqn8EkOO_$k{=r=VNuR`@}rBOk;7K{zmuT?aj@yfBv1E zqU!a6sd~LkXx{qEsD5|B)Of7NgyXGe=&lvn3WeSm)q0K&TQRH`U~XbT^&{=?m8p+? zi*8%ltZ?eAz$_lbgFyM@g?n89-v=`ZKBpA(7)Luv~l{ zaC~my`0cC!c%EIXph5G6!1wG&enJSoh|v&(;cCCkO2Ntgl9i!~TvnGVa#=m#bq#$G zV@p|UFhYT>4LG5QtSwlf$WRHRiD;}nn4y@ggGelru#Vt|Qb9LNBhs)g#_iq}*gZSz zMs>XC2aGyiOouKY3bH<;uqe#>iL%fc`$K1}!3Gge#76$m842Hy0kZD^g^~R4GmOGG z6PSJgRK-DJFk2{w!VACHc;S~BFZ>F^_iQB?q{VD4^+senjM})%sEvD#+PDvxehb@g z-0=tCjz7Q-8ny9+Q5(;Sdr%wiq3^>Nc3(m|We=n)UG|W0Je0?X&>SBVj%QDd+W6F{ zjnBX_C1#<7=2;k_dG?a3j@c_z9p_-5sOlKHWImZ6x+JQOIWx+m03THvx+FL$NPZP$ z1<)Opph7xer>b$+sE{7?JgUvTREr1IvLVpFpSm>iC`OBnX59BNjr%?xaQ|?gT#key znFegtB%TTAe-6)VaDQ&&zRwH9zmJ!Y2jxLt1-#WUsF2{Sph8}jm!Us`wc<4m?yn8* z>JI$=;IDXHVz799pnRJ*q*`d+!np6-0_Der7D@QNo5An{Y3@5&cZ zhYG%!a6DgXG|DyL!$$CRV8ll8_29(DK*1cR=0U*(>%|X&_gco`sWnO^_^%&IZIEW_}86*lvCXEZAOD5`YQgm(@{q6xt=}mDk|tKZmRa{hgp-q8=NW4C=9=OOiVI z1^p{E^hY!->KY~H|F$~I& zi7M{GBCbZKC=nmM--?Tb#^ayJApXS0nGWQ1zjMr1L_ zJ{yqzNRb^?-=m>aj@1)I0fX#|8t;EFn7IX_7?AxsQHpxOL9ayJAN0yY@c!4LYH(4T zs==XGUIe~x4t#$TdgWjG9;y%?=m(-bQ2t|7Av^`v?})xq&rx|mN@Z7&9$a)s7fxh5 zMyc#!Y6AM2nt*;l_A$j!sv(D(naU1=f|=LOD;A@>Tz;_x$iA3Z3ZAZlSOWy#O01;{ zb8!#|z8jRv5IaPi1Zp29P8nVDtWhD)Q}qC<4N!S5E>ZP>xQxp4b>gZ~9!NDHnTWPtO@Ilf9_xyj+8>^LOQ1%hcbd`cQcnvv7yYd(c2<%lkm* z8{`9Yjo&06qI3Kf`3T+Px5>wtXuCr`0bV~TpJH7<<2aFIiw8%hzpc0q^ zok>Z5D}O7cs4o`O$tnIR%5gtp0@4@gcqpQD&YVOP6H5eXQaCp%px}nRmx&Qv` z_s6&fJTHIz#P{#~{EGg48cY57-p^P2*VhdHE{&ysegEfA{QeB42;pb&--vt5N{^pS zZ@z*x%=!ag{Q*+o`_+A4zhA-MpJVtPbnSHe{Z-QccJbx*hJZtdv#+ z%c@YUWCExI3BtpjDc^jUK$AkvOJK3dcuh-OQ?+rIz_Q~HrvX#=UwCNGTH~)}UH9&7 z*y_3;8{c}rFLETgd)c_4q`gv{x>9UMqbqqfhGaT8HhZkFvh(~0V;;V8tatDHOOuX? zlpttI$r;&V*E%{icR=Q8Ba3vI&w6~VlQ&(er%Rh&Y_#*sgY>5+evQ7oYU!udw$AAN zdUD|N!>jl2{&J~K_?J$#D`o3a_-3B02fGLMPWspCZ=W9K?O7`PaPNocjMyhW#ONXHGlmv+v-)*yM1{u zx5cwEb=ti6^J1}iZ&o$WmZbNm?%SqBJoswuvf9^?UBkS6Pn^x9TNei%@4^wQb|mR#aF{yfvTSLGEEmo~juyHkf21zWCaQ~yZO zeOFRfeV=#ZzOH*;?oOSve}xk(I_0Qy@6Q+87Z-aU|6%5&H=9rXurt!hlG}8$(1**i zuWp!oeYOR6E=`zFc*KCyCqge>P3RsJ%CLO+03I;MaE_D<8;{ z_e`6hu?_68W5OO3&6UzWXjjY^x$8W88+Pq=@kd}Zrp|M-s$ojV6 z+z(F*#5^|pQSaj$@0?uFs`RR8sn$*Y@Gk!FMQIoJyB~M|^htTMhF+O^l0W^Hwauz+ z(Xw8jT_@|5Hap9WiE(Oac(U4^O1xaR>1y{zD~B)Iac$_6-KURMDf}kVtDd`COt=~) zJg8aNspf5iW5s#3{LQ9mEn{~*^lI&l)Usk|;civ34ZqPe*E5$b9a{UqmuEGi#jjU+ z#o~kQD`)<^G~(*z_p3F!xAw)fhgH@#U6^X}!RaF#&$v)D&yyqz=AC=FEpp4`;?9fx zQSOHo6Tnd8)ALH=b2` zdgbkx!B4lp+feZ9(`JtbUs!f2p4yk@&=HYY~H+((ZEBD)_qhF?fIXBzY6jvvF>ijNo zreV{kgj^`M`}vU5pFWRB7OTM@W5;iMT=`k+=K1b*eY0|hNK`O+=oRY@?3#RAno8CBHf!^7Vu}6R z-zOi>Q_Y+Fwnq2md`w%neD$d3mbYkIsqWFN3m*3h95OO9g@eNUvp8C1HpG3Q3kR6I%sl`!jHAur1u95OrTdbT^! zD)E*m+%RmbY-9_Gr?a!&HirA7tUF3@rNm`)G#&7N4jK)>MY%dhnaT4`NL znsw~u>~84VB>vI`hRyzC%z(nT2dxe3l8^Vi`uu6NZdIa{kmG2zq0;T9oPhwq=#3e%X|1bWo>aPsaNnTi+sm8uxWeq*l?l2h9$j zIlQPP+N&%1bLUOqp19aJ(wnKDOM{;_J1egD-!2-*H>j8w>6}eV;AU-czS9?w;K^ z+tb6>W=D-RI9rAbS=#+IrQx+c%RcY?^g;9sbmI5jxpi%#rThU<^PuB<>~x8%5;r?V6>B$zf5?$V1(Hb^N9Nm#`kC$q0FnH`9HS2 zaXl>hf$`rC%=wmNL0)%BUiWbs{-jKSPviT1iah>HgRC+s|wR-Zwj}!J^`}iu` zu^eyehppc`B(z$$oiWc`*j^y}?Ybjcc4+a&mofReVGqX)to+WGG3%yd563jR*yVJB zk&3o6luzC8iF%^2fKAR+NjrGkAhejP4 z`poW-X?ey4!7E#2^4wZ~wSnhZvvEH_Z?yae+mNGLNr)VR`v2r{B(-(J}me3FmdSg>hzNOnB#gj_t2QnqAL& zaD1wl9cOO7-+oPniD_EjeNZgItZ82lH%y;sZrsg55ocD)dg*3ZU%X&gi;JZ`_k8`w zvg!qMR2x5UYiPxi zI>}nk4|R8Ud@U+;oY}J1AD+D#Me_Hr*1T1=BbqqUoBex9oh}|5{H^%ksA-AMbG` zT7~6N{pYKr*X741+rMN?y=PX!`S&Lzoe)sw&hnm-pR@D>k|vBeXwiVgiQ-I+RHRUh zl+N7MlhRfWZPKFFhGm!6w_G&3(oNsf;ONVXot@8ZxxcCVNWl)z`LtY*^l@L&EoYH_|Eo`3NA+Fbd&=4B8=LuquYyhY4tMPS!r(>@0K@CzvZ_5)>!L0@7^on zzZxg^k4@jMx@%|paep6=gjuI_5}hNTmH0hepsrr38h3VQ$vXS1U1nXn-G25)g+)in zslATYSFX9uFuuQJak}ZOW{bB2RUS6y&PY>rZx_35Uotf9lj*C86)EQ17hRVxowD|h z`-9Npi76d^!DIP5PlEC+LoH+X+_aMNQF`vDLrO-x+O2csLA_P{w5TP^M%4(+=R!T} z#F7a=o}XyaZ68_|mv}h$>$Lc<=0^3|4w4VkD|_%tS+>5@Zl}Jxy?(;S)hB{0FA2%} z+)HmKAIxGb?E>W6)BMiVhv&yuejGkor6YBRcdV21x#`JEe!0vKK2iPRkoD~9`O)#d zQ{GhDm}yxUKHcN+^J>JFN0o#79C(w;FMhk(yul%L-#AV42Z@{KWvaY$t+eRWc=ygkq-N!tO3M?4{Q`;j(AGj2~INJQTS)T(cdvs4I zg_mr35w%$-Lt8?2Sz@xWYzd3L3iL+u8!#}Ar&+pbQ809(S-14_sUsD})hj}bHYBs@bO-|nU z-{F1S*V~Xei?UG;dYG&-X-QTESyRlXJ9oO+23}YR?k2Xm+ ze6iuuV(+0-xKOvjk0txG<6mqre81n}o^WPc%IcVlqn21@PKv!isK)Pof5mLdzCABv z$80dEPD<wfmEUuX_$Jls4 zjK{~KA$*7XiqQQJ*RL;c94-twruXhgi9F~t?`fnnkIue8%>3c16n#@;<@k5InrH01 z5bvs4m+v{OYxxoFp|GcI-9E-R?702&Tui@F^Yj~se)H?PI#@R6(bDCq;V+Z-->toz z68L;h?(*{sGaXG{Cdc?z-p{OGGQX{FmSd@A%+QAHpkFqt78q+3$8@RgoNl!z%ZZxa zQu!ihS^ZX9uhrjI-_5x0HfY{E>&dB${J(y=8oAzXo>$7Z#JTIN{ajL#@|E1m54o+` zB_H>`@QH85>!OG6UvH?}U6gRj_2rDP_1%_VGg2D%{I)5#@0x#I_A>WD_r2K7jUReb zZ=(IuK1E*YGQwh;kLmTDO92mHF|bfsy-iQ~Y{$ZeuuJ-OWr8tn%RA;lYE{!(2<--Q~;k)64g}5%uGZ9`^oPXz1`Ox0upsHZ}-z)*h$7 zhiQASyqqcKsC>Db93F7(h#w2Dhn+jR(^B$Vrzo`Onw{6vKj9mqo4kHW-;v7G=LK04 z&8@kwdfrajGW`-?ao*r&Q~HEJS@zj`w^g4EcQI}-^Bc9|`vbF=ANGD}&VN22>$%x{ zz4cGc?6=R$sycmg;%)7L})#0uSHIi~+#OK@St*W-n$BBlE zT^M6$j{|1bQ%VLtJraJjBIaZ5YSnE`D%SiukEx}jsFC}HBfZ-SdfQ|snr{w$KE-UC zx7mR6{n;;Mz#pr#_WF+^RkFRtrcW4OrT6a9mwhXUvi3cnHgYcpbbOa));>zij0!k+ z^hM3wUz-Pv)v|rD zgRaYRnr7PiY;Ha~wN9cjg&k7uVv}%npP`m%Zp_7QZX4p8h7Xa~tw>frn&s}CbSJ^8 zPpjU9@MiaD-}7FLBNx`zSDTRmy4nUKpALLL{Fxi`dv1^AL!&O-?6XYGnLa-FL-!GW zmZ8sn9(iq2TvptF%9Z80Q3G{@8ucbCnV4L@-SiDEO#i!)C>tI3DW%KLyxk864;f%c zB!&`(=^b-}%wP4p<3WAszWhywU^}_*&b|{Xq^EQP*N*!g-m8DavP1MKog*91s>{d7 z4Tc|AHc0XJI(|pD$Aj%7Y%>}|zf_!(zwa@ndd4w}(H**dS3C6G*FJmHZ@(jMz2nHE zsdH-*X0{wQp1Gr7Ow#yo%_CxL&nTOYZLYqVWOn%MnaRepEE~sEeq5;Z{_e+i)29V% zikm6BzF+t4T2s(CdAqmFATeOfw`jeGDzah~*@)}oH#~k_JI&wKVzO3EsHLvk6BkG8 zk6m}#_0tuE#E><;JePeu=J_mA?`7|>9ToYH~|*whnMY|R4GP1U+Rrl{nE%S*^Z zOEeB|*POH5)$*5Hmw_L1E>?ujzUp*RI@ta~S^fl>V3D@C_=u8MS!@4|Iv?t0J$f^~ zIo+sg^zhvMOzX*6k`+GG0TaF`Vb$cj*=G`_MXL3zG<%R^{dh*^;BBu>&aSNvF@MHK=a~6D2x*@)*z@?^1!0>cNq=-@U*%@;GrQl2K4)+$ zg=%oWbX)FiZ%#RlKT^LpGDG{v%kQLiZC@4uK^ zakpnsh^6}Ek$2bJo!fG2^xe7I-^Q3{Du0RAch|jO8?|$Ap8d9+1s{z(ri}WdUwnAn zp45P|r*t0-aQB`VFjxI$<{9S$Jzu@9J+{2)$KE?fE}7^o5(}Qp%o*9!<>jxXiDtyp z{+;xs zR7iZ;+d554sYUx#=&a%7`I{<3lG*pJH@3_Q=GmwbgocWEytIr0^Ha!yh&uXX% zzoc!Lw`Fr*bARtiMFsoHLNogjvj-UQjhhYPCyg6A*Zf}HY1Q%5ojYP_wT9Kkf7uo0 z=hnTHF3Zx?a-MuW=$V$yldjX;wmq2mRiAx5ewy%LWK_xBo2&gst-4(0Ixpf@u&(B; zcK#KknEer6xR z_{1xHr#qu3#~X!o(YpCAz3rZb=}e`&s!B7&X)B8iyiO0v^l{kdabm&4QM+8uTcz0? zcdu-Zz8~<$);RgMimkJ$=N#XCgA=a{IhSlc&GB_sda=<_VEO<6UI-r8At+`ZZkbGIJ@VJjPT8GHszmd+Ii~f#D@h-(gMTBrn~nHYYuds zKfupEdiQfbjRj7{_nStT+|oB;UfE2m%XDpyn)v2aHoc%`!vv!beoD@JOMJ?W`n7IN zP8(Ongilw|3H-cedVryJiA_NIyQ(lpN13DJP}gUZlfajo;VmFma|;_g8fu{{F|p&L<6W`^8wgAFr%xRy#$CO15eFUQvg_=BMvgp5neI zu)SY>Q~sEp`SD_x>0@|N_Uk`wy&o6ju~^!x0mPYDP4Ekvc|bj{>N8q&Q2+iMm^p# zAnMS<8NGPRs@*>>FWWr-M38f7aC*mM=Qyq0Ce4wXHMscRjP{g`KdKM@OmM2-A5*tE zEPXm74&38*S}Ylv`{%F$Mpt3xv zQg%Vxz1q=?Em{1KwwGC5o%_DRh|<1ZJF;h_njP_eXTKBX1UotFs%e8&%#3_5H?y?m zgHJo!QBG}pb~Bqdl%0FA*&;1_ck=xF$z%8VZQPphgN_b;_=0|udb@7ps-}(4T?k(3 zJypZZ@BVsVP?6eJ2X^xyp?_bGT3^X)wY=u-=8tqNkA5!MJn`P~S@(`_E){w8(+zd4 zi_}5~crRT(b!NR(y+)!`UUc)b<)pnfMk#BI=B_^a$iQ28q4V zNmH&(IX&&r-kL*Ux@)IQv}m~3ctb~EFSXbAlPc2s^f|L2I4${wVeT0HemSL4VOarY z@hd7@3U(Q;o~a%v_Vt~;%J0-YwRQE~4qGR*)X07;8QZ({u1=0z^K-Yadxy9kjh_;w zGis#!WdlLD{9f0qaP(yx-MJAt8Sh`4cn!WgAiAK}C!O?7)Sp$4G}03Xoet1C`C^2c zTKbTnDf3dy4qwpTZ$Et8#a<`7I5+J3Ib%v(H?s|!cMSSn@F1ajd$-*yRFj@8wH;pN zAFQLjd3Shzh*fN4Sljc56*XF_z5(jRT}zYAj&#?H+x}+v{=KP7YOP*A`?-A1CiBEA zdhVu0)-Fb@Y$M^z${R#tt~JH9EgyX{)F3;Gur< z?4Ug-)DBFuz3pmST0Gm#FXljbkrHX`*|X5aD@=bZoM!T{lxsgPjVwR@!*jad%NrTSA!7={GK`P!=qt!m+*+pTaM2e^_h0PV<=mII-yU>4 zI&!ch-RR}&*d_9^)lsM@|Pu+_!G{&n7_cFS0k)#-5ym42pfIC3Do)pdmHnB)^rT7G&yvXWo4 zw@MrSH8tzcT>t(LZFP?Q_6vS0T)GvX5whh~RmF{si`0W&EFAo8@S8gu&33PMGd-If zSm_ek;Sg-tS?k|gmJqY>Nn23Mv0HAr$4s;rMm0XYVZPzy?Ch@lGgk~32QQ1u(dhE{ zsfYP68^&||%e5nZfBhxB?C4(}@uTqE809n0>?Zamclo zr_a9=-()}j)wm(f;qKGxQ_oh%y)l^mZOO`R{X1@CZEyXke{qOoqpu&yY&6*tr!@IS zw-t|Kf(E_XGU~&okJ)YsK(ByXK?Ct%vOo}~CA~k|O zl@}++J$Jd(&-z?4d2h}Aza2Zb&dsjtGibWpT(wHn>8E1X7Jl=X|NM*>f8GbTYHu&t zE?c<6Oy`w-kIfau*6yw|TD5l<*z~%gO+pl!7 z%kP{UT{CoYZOiiNHD8_-w%y1+o42S(+k4-?X4gw&+9eMz_&Zp6U%R(B+P|T6AoEsZ zZbN@-Z|6&&BFCvFf3J<5rDWpduVnIcHFy7`G<)HH2u41I^r$u&$FlBqJ_i?vnwv5!S7tTmo3 zztDT8^Rg+U&m1d~Ei4X_Jh{=jS~qAVoj-fRi%l0^&HVY`aQVUG?>|Z_?H`iKNp_LZ zZ z`u*-*pAE*>r~eJM(@0sk=)ompW^Gz{Zp)X(^+Qhlxw9zq!Sm1JG=nx9kGQV}Z!-eg zjt(y%44=QfA%B=WZcel9y*F`dA`awk^qy+;u}cPAI@Ksl`Vu*J(Ayho&i3A4tLjEA z4YPYQHD+S@f%PNoqc4ti^6HU$&E{?Djx)8Ng5-^AE03I+xOU$+hnBi0d4=cQN;G;z zynE^ug_njVex7(s`w_|LZ|6SCm-ah?e|CrwI zwfjeA8+C4W*}l;Cc=L*Im18;2%KX1MUA7^s;H=G(cduL4X_rI<%-*#0y2h$8 zNsg%}ul^Yxf8)62+f$JqLvpVd7G!-K>S9n>e*bhowyYrP=8)H}WiG8>_6@%~{~im7{`%}&!=j`KeOgj_2RGTMI&7ZbR9EP3 zIQp|=&}Z4N39?utc0s(pbLXX~=S5%F%~_jQr5WlUQ_wSb^3cnZnp#fJ@xDCD5Phql z-&y@=a5yyYOkFp^%4M1F=JvtnW^VfpsoJug-LG40=F2ta2C2x@S}gWdM)rWr)#RV?Xu@pwyBL+lg#9hnDvl;s{#B`H>E)1?s6gB7J@w*&W!h~&E}Z%?7efD!R$gkx16jMKUAZi#xHHi zjB7hLck9mfx21uzFN}?_`R4iOoKv34T6PLBDA#9PK=^I}fEU399py(y<`^11pmv)g7S2O2r%9!UNhHpy{n z*)h#QJ$uBrM%V6NSg*Tje4BFB+J%dJ9(+Iha?zRxb29Y*eEzSH@Fc3-q|4C2V|H)J4dayErk$T1UURYJ!M5oo-UnYa{qEa~ zciH{w^W=d)f9)RYIJhcr-kH{wb8>%|Z1|D=VX4yZm@Ri6EUoG>_I^y4v-g?DOUU}m z(zVY+f8O#P;o^4lzSV)X+m%fA>dr0q-!AW*)!skPA^T+Acei!Mp6p%y-)e^YO%fa1 zy?@^Lo4g=%?EdL5FE32JW?$_XdoikQQC4SyXNzO*jd}KOZd^Y8t$xPu0{QwuZHu!y z0-d|83S*pJ4gB`$-KSEmx(!KtpLVS~KJLcB37dL#jMR(m8t}_|$We#aZGUb_J`SBU zyTwv7pE$GJ%CTnj(aTrs{;YnkWqI~@zlxk;UJs)WWZ&_08Ln#Gc-kRIb;Y{*ul?8j zHpmOkxf^pO?qz*N>B&v~zx(fuKci7qsolzjTrvLZ^u3_$k!66<@bs!_U9alQ=o=tT z-{upTSFN;a9BgKfaDd+71pIr{_I=@Rnfwy0LWW`&b z8D)K1(;KvYm0qrB7%qF^nEboJ?|riRpp}Wi2hNij_Fv5wY_7{X)o@{MS>&N7v--O> zSNu^f9Nf?Be566itji(R`Pug_`WYVDt~+ID_^-7udv9|*SJgSJ$jGr?xAUmUN~?fx zVcW>|-hr+fv08TdMs$aR<^7R0CuAA-?tA7&J~v!!9esb^AjjETQ|_(HmOSq@d-#^R zUUBlplTOUOf2Yu4#=-qduY_wEm%j6^jWTbv~}2tJ%~zz<%_86WPG9nLVd&tXgH1R2W#VWPdJc+$G&fLsva5 zbTA~({CL{lKKhu_<6V5EUDKrl7Jjc^$@2CENFU2NtqTRzyQ*go3-?8xXNGmPaw$)F5|CC<1HgygA_3wqP z)9s`BYDYRwk1J}auAjOuDA+0cTbV$e7av>i@@xC>YBDQExLKXo8;LFSM(Zk3~ zHOu0|e{LMM;jE{@e9Cy|UzoGi=EYu8^fZQP=WPY;9Qfi`wQ1qLcZl$N{nC`F0t{nr#RD zXUhxz#9klq-`j!{J;rJ7@vZ9bIW6B2j?qVKi2UDU&T*YupS>9# zyZ=5MyT8fZ)4!?BE%{#E+W&L^MKAl?RQ#DII+?B!QeDT%YkWMj|67HPKAuo@+*aiJ z$MJVprTeK`7(%Jj{SrMR{?qzXu5CZ#I_~#>A4FVqE_PnsJKP{=-^G3T?e3Xl|9O39 z^x=Oy$-Z5_Q`-V<)!y?{pWlbo2mM}r!sCArkjQrWIEDPE`5IqY>asU0de@J}PBux) zLhRg)J|qdB>4Xn10)6mhH6Qp1-OvXNA9{qJDYbDDiBt!!WZ|=oo1_y+dvYbLYJGq_ z40Fa4U|vZ&%pti>7Ld2dJ1}>ooNT2QQSQ`Q+N4)bul!zRy?1ElY2MT<)qJ7(O0!L? zQ)iUU7@bKvvvizv+;vvz?AAG|ldtnpr%_i)x0`McU2R=`T?5@gx`TD2bmMf->Za;u z>0Z;V)MNDIdfoI?^?K=P>FMhA(;J{SSkG2(f}We+61|mrk$SoMqJB?(ZTqLTHCSob&9H~zRKuHwpN&HM{}~uEaJw-DVNR;XJ&gMr z8yVXe4^KRnq?DwV6q}TnRGRcEsWqwdobh?n^Rvz`IKS}xvgX0f>zX$jGfS8Eca!9)PyN5@}hk4^$qsNr9Q)SyCoiV98F-Se+P znYuZ;4zEB~{e-&D!=b^VkFI?}MJ_i-*t#70sr+-erTv4IB2FeCQ z|9=(w^shp$1GgDVjA`R;P$50ze#XO$?GleBNs_uG^+-xcx}EeQ>046!|EZ8ivuU$m z^M>Y7sL;XYhs~AEZ@+OZI=>(ksH3y9^8?ffzUV{n0b6Hoh-4J{N&5ZY_dn1X ze?qDRs^kCXtD#aN`MB&;a>HhHL)C`QmUXFh|9%(Wv38q8^8V(Bk8rPz63N?ucd*nq zx}|Pk-3fdTiR4X`L{c*w>|Zk;zOSk=sJZ=S5nMu_3I6v5Ly7S9@d*0vO?RjVgfPx$ce-9gyGlQ;kFE0Mh11gw22hwnr2y*;{N&32syp$5IjC;dX!Do>d{JTjaz$&Bo51QhAs}qHK$FEv+><Bq)!y!Y)2j+$@gN)eakSvn`^G{bnx(r$$do8Sx zy-u>8SPrvM17VHqjj%p;kR%&spoTzlOsFJ`ZDrIUU4sv(#f+M!kj^ zWy5J-dMxb)>mjeFr_%2sE$KC^aa>MsN4d7lR+w3}3zCes!3xC1FgxlV^uKcE5_6w9 z&#Ez>SUK~N?ZMQsi(n1l#gISb!VY4WLiUg|%w<}^4rlMPS)4Z8F1ZFXer&iounw;y zXHRNDA{eZt3Ta@qQb-RW2Er`TKv*X=gm9+M{#zfrleCs3l55yYq` zDll(uEu=s-!g|U%3_%yc+R43O7S~fImsbePH8Al)VTOFVvM^4uo{fG**?Vh82oel6hnp%=(+hP;@b@WIRoJOxl%bB2ws2 zkZ@H6iB~m{b5%$h~lbv;xH$#N`AfM zrR0_5HOvryE_o}&3*Ff`>JF2{KIOth4Y9YVDQbz@!X_bzZxn{|J;kNsa&eWoL2%-a z@+bI%{7;@2ABm5}3b9hyDMX3GVFuG4K8gP$XbTp6i7-PLDR>F<_(OcN;3F*OPx9q_ zDlZde^W6kTK91il1PB|2P+^X+Kv*aE3oH0QAzC;fI0z$zxxy@AH_T1*7LEu#`6m7h zf1gj`e+j(>bHR%L&2Qv`_+|WZ!Bsda92a^Cs)Ae?BE$*S!ZIO*|0_@eAx;s8hzG=V z!Xo~%&`tCc*NZ`dyD(GeDmV)&{7e3WU@RC2JYOJ=m(&yc$a;DkYb5Q2WSNlZIHeqhe1-$V=9j%U@mJ2%*qXbl`M1UGV&!eh|q?~aQ&GvFc}a{}g$C&G;Benc;r1q$<=VJ5OCd5VmpV(EqK4@hQil0Jr& z+%LnN${lnl?JE5$eF!VJ_kdLB{^SVAWO_s%gZ$z%Bt?`%n(+}>q5c@GQ-2gztB)fq z$rI#bc!m6$yhwM#eA<2l&FZimB$TkQb~z6zCvRCjR)kf{dq7638oVxygIVTRh;~@{ z%Ye#y6GiHBt1K2jsoKuC@^8+Eb z>lw^LH-mN8-$2UjYsliZC$v=Z=MWrI#T=DV_?Yyh->RKlut~oL9j6ga_mqYBQ`eWyPWS*&h4_VVW>s z94t&1dI-wGLE(^aScqk>u=#wjpf0Qu)(ETl7+CYmmzBbb^8#d?Wx#5{A0Z{okJt#w zT6ZXQNPJ47<&c=P1~Qbb>6?T)eTyz+ILITq2bn<$OdRCt%w;FDQ|W$?VN(nlFKf6+ zZX0O}nY>pSQ*H)kq&v&h4g5~WKGq=r*7skhWq>MyNN8`6_$M|uf;p6Sb6 zVe*(-b`(3CoySgR_jALz3EWi9lfJ~a@NN8e{u}>;Z)Y6&$9yaQnZGKK;z}`CdI}Qc zsvvdZJ!F2qlU^eX35r+^shw+y@q|Ayj#x)bfQ0w4L>=UB))J>7nf)B`l}IL!_BzX6@8QvX5zPJ!+s2z|TzMZD12Q*EU(t7qAP34xY#Y|f6G z%Z-Jk<jhZBo`w7m3DHQLCmM*e#3$k;tm}Lb66RuHP3Jhsn2UwPier$nAf<*< z45W&mr0XE}vI254Du$g^aJ4kdyfn5-)$z*Xcj>4f;2oOaFz{VSQOUcuz5e zT?=c}tz*Zr8`uf#dUiZyVNc|=*zb@jE#dTG_O}k(L90>?uQ;OB`K#e-sucu33=GsH|WUA!*latFv-?h1EDB<>=Y!`*^+Q5D<^!I$44*zkIMUr9A&#cyFNiEJ{M+Cml5J!xA;nMq_` z5+;N(q~BRlKWQC$1!Kn?W*>1;#57_MmC5#m1iB8!mJE=lG0zAys)>2dDl@TEGq;o5 z1uNk0mbP-y;z-d>v=v9tLCjFePWBA9gKOZva-X z`$!p4qo~o;7&e|g$0oDqsj+Mg`-S~V{UUUT74#tTBh{PIq_o&zb~hW%?t$#D$7~h* zj=C-VMU0?(5Zj1IVmquE+Ji6`Er=aN6j#k9a;c(?XeHW<=Hg^=s5nQQD>{ku#Hr#m z(M|La7mM!VEOCZ7TXYZ|#hH@BLaz#1G&H@g{tKejsnmn~JN&0DcTVnjg!LEMfug^-kyfH!&|M(Keved>`3#wZ2dFEs z9^nFRy7Url0ckj}dI=d|sFD!%yTVNGqfd#9iWEai_RP+%4`Cqs9HAn%G0^DXNRTge}5WAwq~0twl?& zlb7-mVY{#c)*oC72?0yUd*mKglj($%uoaN}mCa>wS0OR~4kT{sO3z94AfZQJngThR zsgR<15i)9q5j`Psa2J&ZdAb*=-Lwi-ON-QVNYY4y#Q!XYr0+16j2bfpQt+%HZ*~S` zV9aDDKz4->vyWK@No+gWhioq7l+9r5AxUEdqX}sbd$|pep|An+5FnME`v9ve)WaGH z6PXZ5p;rL zT*%^tQ;ggOIY5Qv5Xih)O-`pSQF|bVWCEmVct9#f7K~;|$n~@`HG*-V(phcDXlsPT zpdXO+b`!EHMneK=4_Ik}glzqxgc0+In+MsOlbK2;n@xk%yi7=_SLRF!UDAS?MolD( z>D#m=Ig}YqTT%A3wR8<560VT`zlJRk0)=3)R4fx8h!5c--ecu+Af}+ZTut^xt|cER*OnX0&E-R6$+9c5yRstL6Iqq4TGk+Ik+sVH z$OXBITvcu&x08LBJ(N9`_heVg`^%TfedNpKzVemw6>>lMD)~hDWcg_MER@^@S?CvJ zmxv%p+^UiNl6A_Zkgf$Q6vApZ){vy7BiEDbL-zP6`EXd(FauUXEQTEPB*;#`2&?WC z%3jIdLOxfc>?Z^ciE?GRhP)4?mJgN>L#gYK0G17D>Y1`K7*pvc?j({&722SzmlxEk;c42*qABzX$ZzU3=G$6BD^ zF8HQ`J7BaQ2vmdHKq24;6e-vPM)yN_0L2RKgVAFlP`lhyfNn>(A<%Ut3Lbz<0c3RD zLkx_?Nnjif;|YGGpaP6|450hM*ziA4`#e#w6#Nteqj(Z{w~yHiey#xB7q#U}06o?# z1zzCS7)}$6+5mv@KN$E`upL~bzz6(Jfj9WQf@R3N8odQ2_J@6S=4KyfN3;v?OAN&<)1l9pf7#N$BNKijOpxS;@5D0Ej5CCpfuo2v* zAQ=2zK@hlI0qUne6rl6ALqQ1mr-Du3U%+o56!?Q-(RTh~U~ClfG88}`m%_z=909|H zf5^dvg56*egMLp+DL4(L74!l#7(yS+V&EYpQVv7(0rMCFZAZWm24E3GpzSDOh`!)1 z7y|8!3`0O0NzwiR#6++%hJboXRWQU9a90cg?JVtvAx?q2V~7u6RSXdihQ5m+puHhe z{2xRTxF?2yHk3l`5X40=I%a@?{w3AG5GTOBF(mX!NG1OVIUcNqA)#+TPWeB`31A%z zxe~04At!_NFeLOtDXJ+zqT@n!1jrj;R3m^41*19wWFgoPLv9E6!%${mBMb?B3bGXc zL79UGV8{;eKn!I99)uyG&q|Fk)G#o*4WLllnqpu)TmqTv|3J?Nkka@MY9!bKL!mlZ zVkkI|r0915Y8u!IL#2SNF;oMZHVQ+%29L(jp5QSU8a>`v4DAaZhoRARsJ#K&3p@csqwP$> z(CfjdjRATp7_}=vzXzkA0s06S)dir@K1|2Zs16PoSVLI?N%sGM|0hVKGl5x>e}6Bf zv*9wpYy~@F(0{ui^Zp;qF0d1ZaRsB>0A?F_9)_6@cE&KpVDvM9{$mEI`Tt<>%(842vFnC5A=)5uJwsi|Vlo!}bTS#<1x3 z)?irFH~cXyI^MMyHVeEC!=dMZ^%(g7jYPTugI;S$127!4r8E%3*?>1 zU<`-ac@u_1eLMt1YJo!)M{5L!f%1z6j8$U+u;aKfuu#;x~8?hUf(E#lZitB+`8t z^!ik~A48&cIDlbM-#dsQkq=?0@!%K?g`P7GW2npESPX@>a|A;@1IJ-F74T6Ejp}d= zL$3uN$I$2;ipS86-~On5SS= zJAlarpTlsA!04O=n91Pt7^Vh%0mGulOu;Y>;8YCT8+;MNqQ0DlVaI?`O#v45M^rC> z4Fq4oaQ(pP7>)vGU^r)RCWe~?&cbl3z}XlM)$l5Y^9AQ%xaHt$7!K9rIsnJY6@YJG zIMh$`FpMrZAH$<+RPap=lFlrBgZ~@=J(A~ks7`hwy zE{6IHzK5YveePpuRG$(Ijp|UU0QI#p1v2mh3}p&_h#}j+bR^>2Vi{(_-TgTG=pe=s^%0d6U{2}4Z>H)Du!@HY&Bo>N;e!~$?D zhV%iWHUr3U;O`g`wSPN?843P@q0zR`^8!HIgMTWB2cv$8=nni=kO}^yAOQRqL!<3< zV(4_}uM!M;UPrIF5$JskdOiK`HT5nqi9zoV2ucC;34&H|0L&#1p?82~7#jK)A;-{YyUG|Q0u22L!OR19#W0az z=uZd+&M5+Fgka}`RWa-Uuo{MiHX?dp*n!}l7zWNWLLI}TfqP+C^cWf#2Kpw^8^f&! zYht*SU@Z)p2iC@rVPG8$y#cJN0PVjXhC%h#$I$Rxg5Dz`IJEzLG2Aq;0S5h@Aq+9- z{SDDi0rXSCNI?p?KZgDUM#m4(sJ+lJ19TM_wHZJk2BUTY=o+vIhK9aJm}2N!uo;F< z0GnetbW9c)7Wyn!bSl)4qFAN zf7xMZba^<2MQv!0VXlKmU^oZxNDLPQ9);mHf=6S}^AIrx!}@{8VpvqqaSB?&;}u*6 zPry*?!0>#8pwPA^VHkMcASPoNbR9Z2fY}Y6iecQqsEq**wK@77fFr@^u>kabj+lW# z@7su(81x#Gn1w-q+Xz%s1Ugq76?6g5QGjkk?SVj#H&?+8FzT0xOaPrrh+F`*4I&>{ zpx_qRML`jGp@MthMHmV_wkw82*Dc17Xy4H>1Ed$&9m5_0dtlf@;3XI`2JDF;QQeke z$bDe+`v4_^y)l#y*at(QKC=u%y#z1EQ18LM80rmp1%_$`uf))B9ulY@0JH&k6^7OW zug1`*P1j&(R9k-xjgDh2hCT;I{R3bm;Pn{xHh2SuLE8$zFld{B7zW*r`Y6Dl$3}e@ zV9>D!V;EGYO&A8BCt#R~;1d`QwcAMy7Y;t90M-1o0uS&R1?U(PF-$u+ z3B&vXpT#h3;By!f)h!uAeFdM#P~X58FzgL*3PwQfkP2Lc<3cq^!;qW6moOx1lgk(i z)&B~HnFCJ8FzEcqz%Z!qXJQysrz{MM?wgHaTfkQ_JZjGz43FCI8iq&havj5~f^#vv z3K$&&AfWo7J_HC0!TA^tokIl}4)x)i7;yy{^XWG71L6iS>T`hL1a8Ifa6S`l82$wK zJBB|9ZpZLH!9OrO5AMK-s7=sqfQZ`R7e++w^BW_gHu-}Qc7p$6geY()MjQ_39y9}j zKMs~+czFIJ2@DVQB;gJS9v+XRFoHIi#t0T*2E#+YBUy|v1I%HBkzhCmgy01hFv2{r zh~W=`l`uTilI(&Je84h{0Cgti82%(!8N-)@RWN)ixGRQ7zt;^T%mzarLh$H1RgB;W zR>SbnugD%4em5A}8zBUM)iJ_Ga4(Dy3WolK5axh;W6-DdNllEf4y=U{{K2SpfUp9r zgW&_gx)>oEtcMW}fb}tg1Go=H7y<5!5$1voFv2XbAx1#^-wz|8HZ{Ts-r)Wi;RtvD zMnK0p5W_ct2VwX#U}Fq_A8dl*Q^2Me{ukH`BcK|XV+3T>KLEiBY>DBapOS+CxQ<8l zv<8O2!x8uw5{nF5w#s^GeAUrUem!M*R{H*MsL^#2~N}MsNqu#R%x}(76N%UBS*60eL<~KU3egNqImZS#;y-p&RC_vYHV(4w) zr5M%-?1e$&2c$O!y-z}88UF|YqqYHP)Yj&bL=io>TkIu#Y z3Q#{nZ3Cd!nB-v${SAB^BR&BqD0mM(i{Vh8Ifr2c@Fffy^CmB2{*Av}fy)4k`fWM} zy+0tcF?1^U8iqrC^*V-~3C_nz;=riZ0HX)Kg`v^yMHmLv@D7Ik0KSW1(S6G?^hIzh zhHinsml6#6`%FnO=VuUOKy9cl7s;0M{Rk+5=$5fKjah_7m6uL%szYVmLE!KLGWM zDlqze0F5o8_oWClrbwZE1<-gT8b|v#mSzh!Rxklo_Z zzkl!l62UVuB&zK!4Emdo#<~8Db@c)}Vwlt5IT*4U?1UkIgXbzh`#Vp8C)gQ7o&uw` z0mvxu0tK;P7Yw}+tbqLimr?rxWE$8FBSp1!$4JpRxEe#Zf!APYR9k-xi`s52hTQ>H zK!?I*v>yQN3f_s4eg&gi0ODgXFH@*S0D50V9Z_%<9H#(XcT~Z3FnZkoa2zfp zt^ufjBhY;l6y$(W|3IL}JBcAH!Klpu65WQ{9w1T8(f$D>dJNQu0P;0B2}524qq+lh zC-@wOE&wNE&=@dv9)rg0sS6lZ2b_XoIdCe5>jg%S3vlRf1?m$3X8^v0;dn4=dw^{K zU%{|%!RZ)Q51fHvMQ|pDvjk^hIMlxA+ygiZFse1csew_w0Q9%{slCJ>mKdK7?U3XIfs{Jhm=<#kVKy6v5AP8KfUIs4pVWvA@Bf_n#E%2LKwYpsFz3YVbP@=K+3?p~Jx+F!W+D+CP8}1AoNO zZs1QC8eLa|p^t%UF?18S4nw29Uyq@i!JjcS>hGw(0_gt()E5k)2}aLH05=HSh+%($ z(c=P~F&I4-!2Sk*!*Hm*S}<%IxD~^hgVE0brwT^>6(H@wsE-2ZJskA|BSnwbfkFRs zrG8?_QDC$m0D9j>p=|>!I;KAu7S$fL6TqV5?!>UKp^g#^u>~x}p#Oo;1O|=c($HoI z!V64c&^Q4Nzl=cRNi_NyKw}B$b5RI1=7c^wg`h&f@VtPa(BlXg68aP^V#q|W5{3){ zLqA54p8t=t^8k#hO56WAw@wK&nUtBd8D=smKnf)!p%W8A?;s?IkU%J*_aY!oK)?c` zh>ZnX6ch(gu#2LL-F4Tpwgp#R+cwa3T_MB&dG9+30d@c1_pSTmxik0Nd(U~#d-|Ch zsEaW}P&Z@J{o)y#^UysE&6(*5jCM6NklNb&Dqr>imHXrI`jBZdLV+5fojByV% zl`-yyrZL74XgXt5LNgfiw@~;EVIG187^;W#)(o`)Iz0nuYoL@ipw&Y&85QMB4>M{5 zG>cI`hGsMB0cZ}RQeNaTGzQY!Fe>T{>SGb=eb9D{wjK%}wd$35Q1~2SD}ln-2^*yw z{!M5rpzRs0PKdM!V}A&GkZ}x!BF%*33g}_RSqy!YaSVe##yDW_v?GiI`I&YUJdSH` zfj$AA#PKfZbBsy%JI0u#ur*;{1wFwy5a+a)83*z=?G?tc5c(?PSO9&EvEK@X-w-OL z>kY=34Ska_r4SjL856e8*ut3bWn{C3Tn9xsLLz;Blp{hSZhn*@LVf|&7zO$4*BJ%& z_Zy4?JNr$B=3@OQLj=tw`|S*kY5eqTK>aK~$`L{Bzdw$lvf^*WP`>$Hj1D{a=^W61 z2Zh}UT3_{}91;2p&;*9oxBZEXejJK&MCdO;lNp2Jh;l^eu%jROO6adbQH}_M;z0KT z#vUlj5nvDu^}GF)e}MXe zeu^KU_Q2njp)&06#^{rw-5Grml+q9Mx1c>4gZ!};4gpPT!HQ0DU8L6r&_UM>F~x(90S9HRu?|K$-DVdVujU zlzbPMwT$@%bP}Wf z1xk4g42pXlW9CAqFecf2Dnt8>{L>ioF6eZ|2t(@`gZymv35=%2^vl;`sqBL%vEF(~fj)4-s3l8*x8G3a8(_y)R!u?>PQWoTa0 zzl@$tQq8`MiZO9))gY4Dx@v zFEHue*E8nF&>I-r0O*a3%z$oVB-wjALwh~^H!-w+>)*l9`ltV9MtKQ(3q$Lx{+$fX z3Honk6tdec#vp&VjWNrhw=*{KuR9pI7{H%zuE<$v+=qbn>4Fqfdd7t$|MWJH!~-(8G*DO8E_p=b(=<##hiI zj9Csn%9x{}k25CKJx?&^FzAzvNqPGeW9tT`yacxXQ1WSD8v=cnu}y{2vw=?WC7%Ee z%5%yOVE+#K68J5S{{=n4*v~*;X6!#gUt#S3hQ7)eWRuqz`#aFr8TUall~Yr%;oTuyMe~(EhuCouU1b0hAen_5=l-jM4`RA0?E<&{mAn4~p_Y z&|aZ{n^F2g;~Cl~6!0*#zbufzC<~#9i~_p_k{IP{Xfi{4Mgm?&`3mY|XkSSng;D+n zCA$FHGZIK+XiYtk&d~mmKnA0nhWZ)WClY|63AGs7nxVDiK#-yJp8(1vL2Gw`Oh$bk z8fMh>&@4th4$Wp%_&^|sQIA2%)<9hcZNsQ9LfbO5rW9z$s4qbC81))xKBIMk7BHFy zEo5k3JkXv|;ah-J)pU_0NDa)o1rBPjk5!#49)ih zC?0^u+kvhO&GiIOZV4Kn2f8yfZxrak=nAwaL-S05UW`uh?ak2mHBiRr6!*&*8p{UC z8J*Hm!O)mC(1+3Mp_PnI*O4y*eHOGIqf=P&MWD}w4q$Y;ALTL7XFvxrIz4kRqo0Hh zVRY0pfuW54DRda4-v%Ad=$}JJF#4U)k&ONsbQGiC0Uga4l-|o3gUZSn#-Q|$Weh4e zRg6J-RLvOUq2m~1EA$G+m;j}7z_<>2C1Xs4(m7y|571{|(D@q1puDSPjEA9<7~@Uo zWX3oEtz(Q=pi>y*LFiP*cpW;8F&=_WXAJVCdd7GJN_GdP2TFMZj8o8AjF|+T%@}0E zIgCj*nadb|hR$P5vf+HjpgdZ@m~_8|jM)Rah%xDziy4#rm+CKIQoNQjCi(a>#w0r} zXH4?ps~D5)w1P1y?^ZJA-OyExxfXggW8MQ@&6w+nhLR5f^CReWjQJJxddB<^dIMw2 zgx<)QjnHk3jr@8$W9tdMiLsIY?qF~5t=q|>55=!L_ zm{I8Mj7)~!!KkgFcQR@SdKaSxp}QIRALwrwh5T_3BPqZ3GBO29pMj+F_b}26y_b=c z5BD+R@6h`hN%_#gh%?X!7)kbikP$yY_c4<4XFnr;fIh@X$_FZkK>Q7QfRSW-DnCF< zXoQhu`-6IoaPczD2pwBQ0+45ON`2hMHqfp$AF|?mA@H|8NTmvsKv~MqPoKfz8zQ`z_ zKwn~%+n~Q?luw~27~1O^c$uMf0nfU`~gA$(UWBUoqxlDA^8}l~A%TFqc5TWz4=%@+V*}gOc9^vp3KJ=#Pvo z2>lOZl9Js4&Ep5i)_~S;0%S*EBcCE$0~`4o*%8>!|cd=Y9g&R3vF9N|P63fh4K$B0kR$v6)~;}|FEj9@Fq ziFzgIVw|@@-Hh`tXgu&>oHZAkz~BvDBA5u0kRZAb>_XTVL%ocB6x7GyJzpZ20#XtF z2sDim6#sMv?*J3Q4B$r?)LTL1B_Sy7tr@%@Oaz0BSO*O;;st0XBQ`;iuY|KBGz(-S zZe*t%M(l^?GIn}S8^-w*6nRECpM$nzoV%cTjPp)tKI6OtTEIAOhZZu<-O%<7-dQGs z9T-9RPH_S@_*JkIW8V%fV(dGhof$hlmuy#p`2Pl43cBF<9%xt44fc5!+MThFhxTCX z3bZFanN&CFm^h>l5voYS21?@S&(c2?Brjo z8J%puhOupfu4QbL_H~SHH*`H?Yk*$E*vM`h7~7N3jg0L$bQ5EH9eOQe`vY_{WBU-g zg>jJmwla1~>vfES{N;Mao(a8y!MpB6km3r&=TP!XpwM?WF*?O@2V;`|P#ysK%|wv$ z7^q|$%2%LKcuF&%y$r!!46S(v$yb0*_Pm`j;-OTofN?+cPR95Mx`$DyZ0%(fDsy); zN(bmYj9LP{m!Wm+;C+l9gwlP1PCi9;2L|QW1B^j&e2_7J58cPu$d>yVWhnF^Mxi?3 zVTSgY1P?GKrIGRyC}jHxLu>58gN#9Gr#cj<P)d5lW&nGY5s4jIrzjJXQB7%av8Z-6diG|I!}jQI}qDzE}!PC+TZf%YPl zt_PChgL;%u%Al(m+A9%S!zifpLa0j#`kh~B9iwc9u4feTpKBO}uHC>WouC^TC;16o zkNeSY0Yh{>uze1t&%ky+6lI>U9e{3QoZX?@8D|OfCdSztx`T1jvu zPWVvhcZ_os^lipTW#adYb2Ri1jB_}Y@((zv9K6fmO`sxEVFcoysR9l5{0~%Tbi^yu zU1G7dl<8pv(wdpT*q1;P83*i{natSX%bBSR-Zv^T(-?6dG{A_PpdrRS9hwbt z&@Lk_nYoM;S&Ya6QS3}n^^e)6q_%LCodtAfV>E0U{JKYaHOxWr9@L|G^ zJk3P@5>B$!(~J}4JM$UF37dp%jJ++?&e&mtFcM1GQQpH&#@-&9!Z`8#a0X*<2lX@d zd?@Ts*bAY^8^YcJO3wh!1yH&MIG01=%i#wQ2DT1A$k_Wp_c8Vg=zj1p!c;=xpJDi- z6KM{^2gC3`=N9ODjB~RPSz*R;7Zg56NaR-*{G5=8U-l~u-YqM#UuDD-(AOBeZ&qZ# z&WPU&krU52k&ij>Rl<2C6uwG0VgH=2j1xA@>Bcx=!<;h4xfWW^IFZje$PdDaJkEg+ z5KiQCE_|MFj)$VoBAoDt+#!q;w#|Kkv7>&-{R{XK_Q{2Q#W)v2zh<0QLBC;~i=f{! z&J{wmh5r%uF+#L!Wb8A9$VYkzJq`+=BeV~p@I^vz1%;mxMiCUgNEq9o@Nq)IFY>1{ z2A-KejZtCa{OOE}vXEcTXosQnY(V>|^Jg+P*dZV76k!_%oz2LDP?RG=r#QfVgoHok z&tv2v=zPY6&*U#)%(tMh5n-ZyMD#qLcy_zxJgRW+5@T>f58KpB6K20d_L1eusRFGe|u25y{ z>!2EAhmRHN0QqBwUlk@YcHFZt2_(ap#zEmzg#9{bCdkHdM`#{-xmpgY3A zUWGjw`v7P!#@=6u`|o7rAE0+J5_U$>vv6Jym5g&c6zL$Gh(iO?LO9_+4M+>&ya9@| z5Kj0*gTXj&gqno zM7>4WDeiPH*u#N5Xn?I6!Z?OsH^43pu$!X}n!~8CLQxJ0mC^!x5+*$d_9P5?Ry)SF z1e(X#;KvR5jO_@tfU&`s8j$gX?I^T8V?%w_(1D@fM>Zh;2;&5_6C?LQix`PCG*CDo zAA}Y&lG0Sd$cLb0BOsB^hAxbJ7>fFhkd$ut10fGU;TMF2EgE_-@)0Qfgpgz#_z9uV zb-fuS1zN^vbf3!@jr^pXp}qbM6%73jqoEI@(funK?FMLHM)yGbF|-%Ep+BQLp#vE0 z1L#0T{~n4mN$B*f!HoWI=nzIHn-66S_-Mm0MkhNDXAH9I2*!Z_HH>5ovfn7iAfFn| z7-Zwi8H4<83}ete#xe$_qlz);e$|XYX&T2E+o4x5#ti6q#-Q?YC1cElPGHP?p%WSN zI%o}J{tjBpn9o2bF*dqxGGn8-)iDO;;S|Q8@p&Y0v^^^8ezrnmyz zGAP9l*ycfJF&ce8o6#taa{$iUK84O_6tevSM*9Z3kkQG{7cn~J`C`Uc4PC+*yP-=N z<2xul4;U{&moxg0(5o2z3Fr#OxCOeBG3G*7fvZv8DJ(q$sAQWpjGh2p%V=Li*D=Np z=z7MWbYH_5H$yit2Bmu=V+Np`7*jy6W$gDuH#2qzbPMC4vbL46liyN)1G^5TG6q!2 zcPc+X`vOX34A_IvZH!JfraS|B6_k7nDD9w>2B6rX6mOu|pkyPUQXF?O@=NHgjC={Y zi&48kZ)4O}&^?Sw&)v(YROaqx=yyL2_b~K(tcH6To$}&7M(+%!N?$OJG(&pJXK2>?ub2pieVeDU`}K(8wR2 zWz2f$bBx&)`XZyWg;G8Nx}US z=o^edY5kC~wTFJh*vQTwGqw)UGmJuI{5wW1f_~2!L!tj>v;k1EBTy-h|6z1W-%pI* z8G4pcDedPNgY4PJ=q1o7qxFWKXLQ)}!9vE4wC+dz2$=^R#h8DFj%U>E&ZkOvZsa=``YU8n$u5#;1|a)5Su# z`#|S0?kebffcv?JKo>FY!O+Ewdjxa|;~ofI#<&&ea>hLzif0oJ3WIu*aF2wpVO(EB z*MfCIxK2a2GOj0}w=nMh(4F8`T-z7A3*3R@(a<{?H`0CjF2)UeoZbz7gD}XK(|f>P z+#j62n{lDMp1y~19fjV@xSoQdt{_~`LGNc=FG5jQ5U#hN4=^tH|LF%A*Za_Yj0^dD zdOzd(HxzXR;rbVpo{c)f1^+vZ`hjq}pb^HM1U<;Oz0gC9>u=D*j5`2*lyT=nA7k9D zp+^`ueDE}-4Y)f&A7|WMpk!a*qWC_^xai)Le&Biq`ZVK0`96)hfpEPCeU@>Zf|8AZ z>vQNa#`RYy{O|M&`2IWSamLjMB|8GQ1B&{CaC@Ni4B$?Nq7EV4DDS6VX52lY6er;N z0s1QA`W^H&#`O;Lb;g|neS_``U?u>(R zJqUF&u7{v;j0=8orWK>Cgu-tK7vgxv&A9OVGx3b;EEM)7T<4()j9Wqz8Mg+7&z*to z-4quuwydY zdnUxV;Ady(UciO?KNDtL$mcUzjH>~f&A9eMa~Rhn&|JoK4BCcq!OzaLWn3Ra;Rl2Z zd4493aUoC6*;d{RWEiNw^+`k{yBTaVUiYt|y`08P^eL561N-6y=0)9e|Rpft&KVH{(u& zmNBkF(90NPwKzlh1YBQ1;iH6`Y}SWylRYXyUwlu`?+5xL{5{YCjOzjDK*kk;4q{x- zLkBaiZ=pjN*FT{{8P_@JFvhJzhcj*yI)ZUiIT*>f6QH9QH`#tPia>h;mJBD#n zxf#p2DIcmB4e2>k&6pH+9OF6xy@GN59Xg(IMWI(RuD78R7`Gofk#T20YZ%wN&|1dz z0dx|W47+^-tz+EeBU2bR`NCAj^%-;;<0e0y&bYpS)-!GcI)iaje$HfEuR~`st`DKJ z8Tle~4&x39@m-K{9f#6q;Cca?$+$j(h8fo%q3~(K-3FS?xO1Q}RK(%CTt-D%`HsQ? zH+=_xBHVNx${*o+7K*eHuBV~-jOz_(0pogAh_mofLWK{Xl|aFVV^Eb*lb{--BHxis zA_~WLqD)1D0O_*73`M%4NRI>l5IxU04+?SqKL8JKqO6^VFA`3~?R+DHze^;|H5Nkd zhpx2{_B7}^3t`WIuD1|!8T1+pAx}a#SP1z7bdv?WXH=TkS_q{Uy4gY~lc8HIgfa!X z)q>uKDltAM=>4hEyxxM|g(|VGMhN9O=#3Uac@4VFLZ}qC?H0nk6?&6}Q0t*PEQDGE zz1c#ji=nqz2$i0>(?X~xpmhDM$P2n|mj%6>Lz=f)&~MYFdAkMu?p&I8SkQ0irFo}? z(9y-k@4E>4-Mln+ThMROCDtqn`u({y_gDy99CWXRu(_ajTL}Fy^d1XgNa(#5!h8vO zpM@~`LhrW_#&T$bg)o*vAFvR{ROo{i!dMC2XCVyAL(1>{7{B}geaJ!>KSCe25XMi? z0~W$K2Ytjs82^MuEQFQkhj5H~*G73k@ji^>tD%ot2pjp%V-~__1wCRR?1Q04Erfj- z^l=MeCm(piLfE3vCoP2Wl#uVhUO?#G@EUc^g5FOr-_f z`ICx;P?tbe3wl3_Jc&31dQX5ni8up#H-S89SO^>JbJDaBI^uBBW+4o^kKIBTe}^KC zK-e0gP77h644#C2fG}Q!(!Ee0nDl(u0SMbhDC_`)LJB(odXI@b>9L@9Gsu$(7DAZ@ zg?#|M8$zCheSpxBM<o0bx+O$fjt26tYLaLKyIcli3#Zj%0Zf`2mD>3Yu#n^fAyj7Q&bgZEGP6 z_|(aE7Q#3!>EUA`5z_pgh&tLTIy~#TN8VTzRU*LTD(v zr%Em8y`l0{7Ym_Z35A~ldgqNi1>Xhqjvsldy9K>7K%VMhA@ohqo)$vg2km7c4CL3T z-WK#;d3mbLLYVNQQHdTupvKB^|cTx%HpYh z7W8)r1*pjCPS} zj+iB8X*K9?+;ncGwj#QT6yL_aj~+PxPSY7a67S)^@nXEb8gcqauH`Gu+2>E{^W@rS zxpjRfVg0A0zo}2s9u(qyE`6_mf>57m51#*$KR4ft!ktjjd*kc%)d+V&{r>!y(LC#Z zTg6sUDoXV|$d?s*C*1Ed>wbIApD`cO_uzikXN2N#_u_tEXdj)wB6hzoa6kTR-7g~Y zgAs9gRb_QBIOuWV8Zjthj2c%J>6{*EQ(ZGPxaIPyh>~6V1pY3tm^?W=DLoX5i0X)_ z2=_T8q^PJVFNnzeNU&yVK}5+9hr*$Ph?*a)JFF%pi}H#{Vnwi~ru?9iR8f8~TdjyF z6{D90BXMCIRMgf*wBgGRDT;#IL_$+CLiFt+SF$Y62;wkYekf5U;#@c)hF49gKID}Y zvLK@6N7TGXazzzAFXFAJhy_Rw)&(QShDWrVafjN-Ruz?#Dhxfw3mtW-t_~jL(1=kRd=>i?jC7#$9Vq0n;Z;FcV@qu?;uv04gKvU# z&Ot{dbW~E4UQ=CNoepb7;wmOb#K@|M7({_X_?$i{5}>2NpxVdd#bgTcxGpADSJ%~6 zM`T`gbu0zd!8)WTTwYxe(es0q!HAY!i!_-P!>b}@xIAJDmm>#oNlih-u*?lh1nUl( zlgfj1l9HRw(WieSW=-Yfh~6%QlNG@&!7X^!!47&hEHt93W_Wt-$m*(ab*MTRDH~OV zbLnK4SX2rkW`4w8k#|TaoXs|T43~%DF5&Xph%#wvL{3ITB4)dSh&?|@v2`I?nwW$j zh;LaG6vuawd5A&iSe?MOsY>FDr;7<_{WCHE0xnnI6LTiPrZ?`3FTp#pP886A~&SvbH?p z$)h3%*D62grhnq`Pegi=7i#wKs)JMxVXpEm$YDG-zFjDc%bJei4|JX`K}n{w)ksM{ z#J?ZDy*RUeB?~doN({ru6%o1{qapHCLF zIGNLYasi*jZvrCjykAWx67?7-mD1TVeIcYThY zV}G$ghHJZosbc@HA>fV$kxorne9&c7lE+0lwmYcH;U! z<8FMP%AZ#gsC|Ahw)DO5J}T&ceR_qv9PBKUDXk^22NLj$*VLe})OIO|l;#)udK5&u z{QrW&u_q&VS7ed!W(Ny{{isTSX$NfC(l6W(RZtZgKGeEsk-A8koCFJZL*3<#_z+A( zb((DjI~XU*BhHGvDO(D|!C;RqxM%l^g9Quuei0*F-V{0*si8WhY(&*zEvN_64{JGk zT6Hi=-x+Air~JfSbr39+7n`~AX05D}`94OAk?3>^-K7UYtc;EuJ_T2Ula{2QxDFFXs* z(xb^1LBv4MiP<9D17_;kd@ABVWfBbb4fmsmQZDvtI%g$@Egcb~s|tfX(4bT775)-8 zZO)2_k&RCSaH_fQ!kPGU&#MW?atk$oL8N!n(=6-9va5zVIF}~5DT~Wce-x4x`$l{f zRm0QKrUrXd7ar^&lTdCiyXfr5^x+qsEx+h&)3q({cj*-s`H^mUEdw-N)bzkU`H}8< zTi|}=V_Q(te=Tmv(85RuM6uFJGP!XMTfP<@v~o^5xpNpLtq>)T6Vx~VpaX3UdB5fP z|2voN_e&1;fAM0nH`QZ3!d=osE&L@^9gAl_RKVTxnyk_vpStITD9^|)vD7r1U;s>z z%*7tP6O{18!bk~9`M_WKeh_XalM*AP2s1c8(iJj5SV#b zKIV}6;$S!qBpr;%KP0VhM&iKwW)y|#hf|{|3>{ofVd!8Cg`tD7`Ivta6*#KG5!!Yf zRp%eZ#0(vc!x4XX1%;FJ-FOOT9bHM`tfL7O&N`Y%&#XkS8hR!j)Y3EQU=lr(4klBW z{y3KBVAlab+*Q z-#R{|0PFdX0$hWzy_@g1!TMy~ZzCU4@J)P3!LP-Yz4?Bd`H%u^;X?|rHUE&)>e)t& z^h27WR$|bEwye55FJhY#QL~0GYiiI7;7juSdErKfzYe3uB-18TVT$%;7+333RfIZC zC{kkDT2bUg4M%bqz1KuDHPg@$R`nQ$FR#cG@<>ppMb56%2Aj4Ye>C3v&v@fr-1n37 zU&;?KUbcxOG2n=bZ=@VNh`RMnm$CRs#1&R%uAzF!GncsowM2(`QphU@96{Td77@C=odErq- zopgF)Q72_&|4`+as_3JYq5f54^pVjx`F&lT^)GIXd#^tKw!T>3C(=c0F{{j#nv$HP znFE~;)6kRwgCd=f?lh;aNKJ)FmB|i6!LWX!-KLo4WPy}t(jk2-O)-rEBG_0`sZ8W+ zhE9#10viH~!(&XXZElpIoX6D^8Ei6P)qEp(jI7e}A2kprXKADa4` z==%ri?u~vT`_x4rm2>Z%zFX$qJ^N1CIyyM|{uJ3(oe=$KYGX7cKW&VL<;kfsyIy`4 ziuSVpsgGi+LKRET7ibRsjK~+AMGx^xS!%kp8HLicbuDO*mrxs+w{3b%ofukOT^5&+K#&5VOyI`uZ6u)&=jS1h%il9S%`Y_xsC{;+jP~o1~;}z zou;-gz6Y+UYrbX{%vP2XAK$G@=T3$BZE`}bQ+@G?@rg+}VW%?{7Br|#QlX?0>XkAq zfA(GHoNy-Duc%WAEZBT(?5W&3qyDDu$o`QlG0YhxK}he&X#xH0e<0?FM7?DrlRN**f4-?I?5bVs&u( zr6-KoiE6&Atx$DMS2dVWx9YL!LktN=tHG0MtwU5QQqGc1o}G&e3|%!^TtJS=;ZYo9 zZ7g_Mc5_GtZ*Ed_%Yi8zvgATIc(SW+pU$1y7q-bQ$c{IW6QNLXu1ksKMJf3)=SVVg z$dn}|r6oDJxy7Mm6CZuP=6vw_N=gzXyEpPjpBvpYzIfS=n#C_HyluI6RK?VTbEe-j zp=!$@Srr^)Q+%$u^4=Z(DFyc*ZCo~C(x@qA)!Egx^P}T>jv2OT)P$=$PHQ)!WX93O zn;z-pt6nu^+8tHHudA-xG%hoGeY-6^?0y+PW{h|GqrIMUs3)s7%p2ZkUUgRf+D5m0 zEpO@g@!LlAMM@>r+0wA;?6fjptQnybWF@MtJ&H4hn}woI(m?GzrYcrDBNVP*9{ogl z7Vp$Z5XNE5EO*M2g$fLk04pRLiYF<->r@zvy72mjoUH^#w9>s39a!B2wKZ%CO zE?l^4is)R{(dSL@#J7qw(iI$ROW=m4}&Xd5*J4abtCB)2;$B`L+7=uS-2oxVJo zn_KGBOG|xbVm6LS)l#48!-1*%c*BI$x1vefTd5OzMt>N;{S#l`(PR3D9G`3-uN*4= zA@TlQyY5f?L1w!b zh_X@~&6Xd@J-1Q*uR_0OJL2aQiDg)gkXCJqnlURg7KsbplISYKnICkv47G zWLaG+@?)wqxO7kro?vz^L^a}4l9K{TQKw#t*1qS8r&g_cVnvT0E1p=j>ZuhyKS=96 zYvhPoz0;s0MnaVXJEG@K%%6WkYCCpF?d1gvUXGsIvG9{a_4S87S-9}yNPT_eV?^Xm zq-;Cl9WSmZv&Y$OrY05YAW~KAL9=op^m(wcS_^B!&`ni%a}Oq~MNp|z$LE*AmStWV zOcfHDbfK=d`YLvhFFU)9RgjRRlB|-FVypEuldU|vP{Aw043!pXp>w^D$@u-lA70jV zY+9?LtPX=#y}5Mbn~R^0KJ}(DF>A`^JHLJQz{H#PX2*YKPwUZh%$5;T4lmsp{W1F0 z?g;r6>;S(aJ6tBV94W|G485uMUMx@}>4vsO*ih(g)QeZitJa_jQ_=aWM?+}VqMu>) zI4*c!(XU=-qUUB(>lV8nH5_$OtnNmfH#C6E(ZM5N8PkemMN$wVHRKf z@lw+mW9ce1-8E+4e7R}mbF+ud@6@(Zw;%BZ#_t$eH?Po8R!dpBaNL+>y>b%Y(7YYW zhprnocK7Us59Mk7v3A*_Y&tOBs6G;BD4N>j;$A~nGZV6Pm5Kht1^d}fn$x^l|45eh~5+} zlDne)>iYj$%s8psC2%0_J>+!|4fWr)>h z#Oif(cEbzwX&6m$_?Fq`$nqp-BhyT$pZbMbtS@qLrcuq((SyN@2bqXdTKJmi`DbPv zoH4M}6EudVP72T7Iqa2NI?b=^t@;|TQb(-n^WJ~t(_7zPF?fD)m+q$Wv@dJguKo|- zmzp-S7geIQ-}vd)Eq_}7yqqJozUxly@?O4w z%-Uo1Z{K!p`OY4y?6z{!R)o+o2wOV#W(j1ES>xOEw?_{rG2og zZ|i%@CO05$Qe2Jv8-@I{i<{9rpiV(0sZjOQ3e}Sjl?kek1@!$yb}NCzQP-2|)f7H0 zP-K60o*LQMR8-fVmpBs}+&SVHq;f)N7*} zYd%o!ZQLB1uD+}N^=xS$?d>P!bZXG3%_>0M(i)@4ZlZsgy<@wKIO=dC85ED?7P%8^ zsx068#RlwXmy&|Quy)A+QRnjOCw+hO)V}pS4(;weCVS#zYlqCbvgpe7)qdaT z6(g2BH1WzklgF;BPV-&9MQMNa_2n~q&Ah&Q;l{@MN6j45somN+*E}+CS&RCB{Bp`; zhLl=Uid&ng2vELJ?>7%4kO^w7fk9loiX8Ccdi00%`U@eO&d*ZO)k;dkex$6-)V67D z(T7aQ$qw0I5?`^;i=J+Ctx3f!AvsCW5<6iOK#d61oMe=rD%XE{dosJ`hEKO_IkoQgR? zbt!%5-3>?lhF;RQRKANv6-md#Zx_SSk6wkE(m0Ify;EMcESAYq8dJ}9)?Pn*R2wi{ zzP}p(k4}%GW|Ftd*~V101|`K5nPsiv7$|boP9Q~8?yxOYQI%oTwYE%1-D(iK9d4<7 zx;okldsQxZwk~PBf@kCZR^S16_UlS+wC!qntK79Zimktte^bwYsg6PU%n)tFlp{7> zL7ZYmlX^9%VW^7GrO;dUYII$zMSC$zgSS$z=AvLty_ySwFgnP}&d&6CxjMCGUp&1~ zq2`3mm`jkideM9|JBT`F@X}4)7hbk`;%(QBPP3H`>$`pK^5^G8j}Ph`oA`^b-8Owd z+Gnz@cii?VH(vL)(z#p5@jb@fclPtn0|s?2?mv*yw=;Ue_!@iD28b#-X;35)RgT*$ z?TSrm_O4wNG-!iSVM&$k1Cgw(qE1?ZUea7op}sCPzOK3L;USj~8g^6vdE3TpKT?}NuONGY;%l85 zNc4oe70jB{tM;BZW<3)9=)J}J#<6Iy46&p#KhWaZUdC`Ye!6Z#+-Xvexx>f$j= z9fyIDDX~R2R#W6O&tB;Q-Fu@RE{(Zt)XL~lVivM_FrU>NP(cvPc%u#EE1H5)0?VAE zvnsN^nJHN|XTX}R4I@ox!aSX=$*fLZYj%q&46C6Jdpf|msG+9@jy};xZByNE9v#tm ze3myk**i{7WuKZ1PW|d9J zaBF?sercQT0o7$uSI|K-?Kc2Rkv> z>WpU7WM8bjP}!on>KyzZDOZx?1r?c+Qj7{8)fXSBe<=FH(Fu>riudY{u3Qn2sdnG& zAzM$blS$w8AEyS*_DkCH?#g*-u-=sGG>XB6)FCZPqJFl-5XedueNDN1G?arJ z)@f9O4z_kl7HgQ0P;$ay5Ar=hFCY3~^z`$MCuY$=aF!DPz8wA1SL^?A_nr6e{$`zW zXycE^=eNwc?U;glU#g4FPF0k}63>&CA7e5_w3SsDRUw`x@(U%+PO*0)ze>>Vq6fA> zI83|Uv4CdDP2D!_Vox?EVm5X%Rpva2YjwA&H(uJ}YSUpyrvHyGr`B6aG$AFP z>VBGlxll+g#|)#hf;pMeB78#Wv_6q{PCxtM49dzG>SrrjORp>1M;5t!$ z!3B2QSlD5|ORj8sNS#RwQm)*bHl8HQeDMxyH>nsm+o#EP7{6H6`30+LzU6*3!4(}W zySltGaq84W>1!1|k?h^v_!q09Tcv)kr9SQUHh!D#*cL6i(U}&Fb7v^Fs+QK|x=W`$ z&h5prvczN*&H_Z(Y%6v2^)Q8Xp<>56uxhSBZw6gJ6LprRDYS%(YE0Ly3ay?7ji}gT z0TomT3I^TH7sP^LRRk4#f>o2{c&v&YYZA$x4%AinuN7KRtlHPdPxmITcyaE2)M^tC z%OPJZIyPZNNKJA1=k?$5=hf2x&p!1gETMRn-VT`-AI)vscFAi}yWxFVakwBg`q<+M zsYYDe&Cws9i+&J&rnS5+K24(k)l@&|5%^J&SXt(6m#XMSI2k>m_I6AP7R6(!w#g)6 z^pO;4Q>HoXrmE}4L9|e}!x&A9l$j@urCO11V zl#t>{O5zpXrZN=54CbYIPir{ISc2z9kUvoFm=Kit5cisQhbS7oQk(B+;Z9(wJ-b zFo~+wbxfE8LN`L-sDwH zZu_&SA#O$IPpS_gZhleqm=6P$rl)2__OYNTQa8gdt8mw40=Lw*u)nb}t=7_5gElu{0ed+%46(fxm1xDA_*DXo*u2uZH zVUvD$WAVO!Z%p?qa>7roy&t@q|5N~Qr(-H3ivAE2a6HZn&J+HM#H8o5(%Qyu? zuqI~)ihdX)F)!y7D=(g@M0qBG*e_1-_!{RiC66w3^C>G+X-h+n$CK=fr`htR zh|942rHsWafVBjiY@IZYH=acrqHU)}yCPeom8dIFcgWY|Lj3psTxyx;a?}^}#7bUR zOu@HSU_)4m z(s{PMw&GmE&k|O2_9yirE~xMjh5GnN(`u|KXa3*T+%RHkx#pI2(UKc_xYTdQ$}~Fe z3Hs)Vkso%x68*yiS4MugZPyQxiGQnk?#7#+pD^LM?KdBrs2sQ@dhWOLo{@du+$pu+ z&YgE6diK@^nRxGg(ZAnwZ}i&+99wawKG8_z1C7}xm$|dB+NV;l#7(twEGtmF;HEG& zR(Mq{Hu)dxzSq+BtPD(wW6gjTd14_kc(Ho-bqZQ?M^@)h8;`eX*5Am@MD_UdDkZ4# zBzZ?qCvTCrL>sy7%q?wdZOPtE)L%iitdQTxOquv@!;2Y%^%4HP&#BMfdSVjB718xS z%bO?va!Z5KxWXy#Loz9EqSx!UBX0tD>EfI+XA;es!R@Rnger#&^yf7!#9+Nj!3-8A zyU=^#L6|jhb>TFPVkE5;;I`GcftX~4A6Z?N9BeH`o1E5#!NL@8D<}RGT)=dC^Cadt za+>o}qj^@tw1%Cf$u31AGiirFQ|4+(e>{9oGJK3>eQ{y4!y6bzaaJeR} zi{5?n^AoQ;cIz!KOsF{?z5dDFDNo1_FYlDbZ|5G1KJn_~(RW_-FT5r}t5Q8~dHg|{ zyl1aWdXTbwPxNd24jOUBJ|Vp$^1FlRCAOEPWFXZkSk}o#^_dtSr|Pnh8o5};w?PLF z3!BOs>>m=;a$uJEf(IzrH)KQ0VvS96nu21J5xYY3HLy{g70?==rg!QnMYocUy*l+M z$ZzdWPfPGPOwmDhaKKc0W}yy4{8EF{NwWs>0iy^DcrY4G2zJQK48vKpef29Ua@+L1 zdRDNjit&3?jdg6iU&~fCw~d`RvvXQn@vNFwgBwfsA8B|YzrCst2=!gKW%I(mTD0J2 zzO(nWnVW|u`7WQeXiin<@iI;M>gs#f-_ZCLdJR;@)sNsuMKq4jlsej>3r>_p-UY@JA3rDs+I3;n)y`4;?^?B9zSyAiW_!p zdGa;6cG|HOMX7S;S8*A7eCss}`b`;E)aHtf)317Ta(iF&`4^oDhBIsP^a(4A`#wVL zRS0#)KKM|o*of5%?n}|IxHbCy@E5Uw`f}DNiCZbF=U0o>3Lg6hW5JX)zZR}6&>RH6 z{Y7nrUQKhOU58~2j1D{rS=geN8E=gaFe1W$sAZKB$~3LjrhUu(W_t9aWbgXMch`H9 zqaOs-jHTLU&37z(c1MexpB+9X`yp+;Q9f26ZMkAhnKK>}ETK5``I;J~RP5SRG1Y}8 z*jk)zIjr$nevi|-ifq1UB;(1+31Ku7yfBhU(n6>eF6o|A#opXs)K{Pj5&bTDzVYA5 z-r0&B?QY4=8*aFjT4DJNw@c@KIQQ3opH>}K8^pb*yk$qG&U}2n@@{jxBt;U^)Ej9^ z5`!?Ra<#Hy9t{;V3OP+T#-ZyZ(dlj3Rr)WRPpTiwNK8;^SvEs3|**}6-*Dp5n6QTT==MnjeL$;AzCU#|6v0KTT)$p)aOe~NU-LN zD5fnUn}{jTMDDnn6IMk}?2PV<9=thvXsLYnDwjk3US76a4*X1wk6$bA){T^(PHS_| zZr9x@RaJ2^&2_F$YkT&H`V8i7<6`kgd5Bs*Gqfx@9P}n4BDvY=9(SD0RP30ei&duy z7&&0qE;?ry(oz4W8K4^QLaM7HvGU zBtF3&{b%$=(@b3v9kN2+8Qr=GL+@16{7EJ`6WsOk;cKGT$Q`Sz;*>PEd?-59k*?4@ zF19-LN75WJBQCmHuE4;~RaKSL_^-w-sgyTJ+uxD4Al74B1yYkdZfn%&G30V-dJ{AR8*VO)Z|3k-{(f_yvgeu-(QWM=hCI}FVj*` z=Z9oZdxCqR+@l*wjboyf7fP?ZQ+bJ!pr*w&eyw<{Bp~j?5%+9kxS3E1{+uiOG@x0WH#;Yk+CW*S;j^g1J4=CPmSNE*#pbf7aMh@pDRHPFCVAv z4nHgVO*lK<>;60Lg*@>^&!{%+K{rLOLG)ADGB_g5*H1=+V`bVTDn2Q&80KCTn)MRa zdLqrrs@PgWtN5`MMQd9|NVP@X@^tih**kj1czpa%=n7Feraf$5!``aM;hNq_-)=+< ztSxMIe2DE}dz5$4>0w!iS?};*vXaj1L;3kEFeS2O38z2cmDS6MR4Uk^Twpw_6=$-A@sHu~zy|tUtHGe)#7G zMRH(uOv0&3yS7;_1>FJ#JBV9+fwAUbtPD~x>>O5fO!T88s#q(hGqGJ5+fA$~a_dmi z+Q3CShp`4jJBOPuK=Ts|-(p8_dRZ=R{VO3a-8<}Po0A>R%ZiP3dE2nH@4a*9;?C%z zVNblZJYPa1F_PA8u!ma)@`gVHmqBY&;-qCi&cdtBGJj8uVMu#;oO<8?im(+yG z(IZ>#8J;>~K#!Z2x9-#a=CxfXRbQTOmZrU$WE@J)89lRT*w*UC7ZOvYF<(x~tgP&p z(|cw)>Uz8>Ma$E_z?yzQY%6nNn4&mo{R}0IhANpbUM4yqv?fBfz)uM@O{M-9min6v z(!39^H5^FUKr^u$emNNC3;s(8o)55BWQP+xO`SBXOL)Sxqwu0FKOSCT%DXCh?#<{2 zcSzgF@Ll6J9l7%DdvD3CZ=KpH^~fuV|Mq{~ppWb`tns*~`?~iVUU0?y8~?Cy^QY@K9UWAd*(tGo?iK6%4!(6>zjft#t zF|EQwEKNh^7=}`dY-*7?v;cUCbGJnzHT>vD!RELB&jlzs|0S&TdxfT(QwI_oO%kwB zW_2LZ(2|8(cBrkLp*|lb6Ua>L*V@qNvwe40PrGGe;exLs9T%ohZst`*pG})quIihDzMYXTG&IcyLCo-a-zEx{@`7jOrwFNe&JUY$w^6O$l8rt zg8l1gD`|Ly>4MP84>qp+aO-o&l*Z|yq1Q~VYN#A@B6@>-p?2z&8&c=~dHqjxzvHiss;7)n`#(T zPBb-w6+hFo9D6pks@cd^+T}s}Auqk+LO3ju^{ebs(y0Tc`Ev`r@JaNq!>Cv>n~d~& zeF1Cq%T;cvwUM3wrDfd^R<Vea*(g>WZ#Q$4^?*TOFBQ)qU2j z18ix%3RNRMv&WH@}+Fr?rZ4g8q259!kM&N zc;*lre%jDt4x4)xnbGCCoB^wN@5`E9GBMU(Pwlg7yGCF_cp>`6};Kip7K5|#L>g_)_1 zHK+sNXlgs{EmZN_eBBFn?QVQ2F$Ft{@ITZC_$L>A9w+vMpr@K7H67bAtla_BTMN^c zQ#5gF$e7ghqwdu8$><93mM?|o;0DoqYsy0ZkG=PRuj07&fOmH9-lnT^i+a~p@9pX$ z)UFCb2uTPbgd~JOAV3sRjcLX}0C$XV!8W#Wf=P@^9O5`GiPK!-U?*`B$Io^g$7zmJ z9VaizOKfTR{%3Y?Q=#O&@B6*)_j@*y?wy^Tojo&W=FB<&bB?bnRt>17XnszqNP?;p zCR-h(&jVO8@sHf^c>SO@FI|h&dss`&PEgR6o~H4rI!!S(;dAK5ffme3D0^T`zTQ$^ zJ|Cd0mUT+`6|WG~s|JBn5Pd zi^UE|oJhn0a#!0^{KVl*D2l{?Dp}%vmGGLd;*ez$e5zJK6F``Q2-rh$5`)&pTZv!C zLDCkNozAz>Aj}z9)4;b?fE3-)ba1ElWKXuZ)c1)BNpsdcIaIf-GWq4ooa5u&z8C!@ zwdTZ_duW+^cKIyxjVVuEEgfw_G%+sUlz>lCh0l3xR`5x3VqBEYU`(ZBtyjXy0-L~4 z2;V%-fTOaD(WoxIeq)cyDCrsx7{Mss8>ZHUO;%-oAi4Zn>%&=EU7D#?H0chEXr9%t$Pm+LEV*k;$BEQ)uN);YRmJ7P-neGCA z%cq^NmjoQZHRh)oEaoU-qhZPS;Z2f4H)Hci4AuP*+x&f@i8Nsu);K}`Y3v>K!Y!V# z@`7AR45>-9NX+b20^|vBgy`qd=tseOt+?|r_xVExYy=G`X&ekfveh-0-D2-Din|Z)?|DArxjOE-h#DdyNmLA zl)9e0u9dT|b>d?Q*!Z8_h1++z`ZmSoTzh@R#%;L}@i`}(mv2jnofz~EuNb{#G|S;% z4aJumJJux3&RJR(*I2UZc=N=0&h?yzx)q>dws5t_k{Bn7hV)Po0wqSA)~-ySl@31A z#KjyoxuM-8!!WBE&ADOyaw>AVg=J^a3T91*X^#*Vs@`Q9a&hW#DMQ-?Rw&gyw%Gm+ zv$vP4#5neGch8sSUUy5ZtzX>&B8OI|#!Y(Zl8%S&oOo5+0JuD;bKrf$u~QN*8mDqv z<~0Ej%ODiOnb_hrT(lnRt5=kdg2{x&)W6?5cD-}r&YxVp^WD4dd28DnRl5h4@1Bj{ zgS)HPACLdx#v6Wr`)z-`>6SmJ*no?wh=Zmif$$n;7v0qBp_fB92~oY8b);uHZILlPa}S+0{0bj+ znT+%uk<=Nf$|_~o{PuKVe_@;*a@)$hqvM^;r`y+mapU*u_YanDDd_IndVDS$|I9~+ zEe7Q^dz2WuU_q#ysXK^iR55SgFP~a}ZbN4DL|w{NUqxqVU&T8$u%hXIifVQhlo*jwk@Ijx}V?h2}`$y zfBaW_Bs}9k#L3+H>*v;eWjopB7_c^iWZ~nn-D6LR#4!r-w4ibog)qE^*XHj;RS{|% zD3Y+x)AFVt!f0SfGz&Lr-LZ{dq83Tow24Pq)6@8gp8@0^v{Gohn>uXKlBLt`$}Cyy z%DY|>AYk92bT2trk(#@8FssoMXk`;hl16G;#qQ ziezwhYD4cZyJ-7`+E7Q(ahW{R@e&$-2q7SHo%TGZ>M-gg^bA?q*M`DwgR`;$rv~$bOZ+@}=XtMbFF>6wLb)lyw zyJY9^U5{pkvWUkW87pf#msVD68@Tb)sR$`cUfNYJx5?8j3!Y%lrtg zZY})a!x=+m2cg4B0U6Q=9HO4U`Q*m}OU_N)+1)KZl5=n8_H~DD8R%!J%6na1`5pGW zsKU|R;@QTAxBt0j;>A6yV?aIXL(YW{__~-Z+yy;3z#kGEoFM+t!bs9zI`o;UHh{!9 zbi@m%B}(!o#ji9afc3CkZCJ=rmcf>w7)SrM>}kGr_G-3Jx6l~bCt=pSMFx=@6PdAuz^>EEIFzyB*jFDbJ&kFubz1Z%!_xW&UYvmeuYz2zX5wcFh!^%NkWq z9#Nj!%3fLtkCXSu4L0ilTO-|U89ylPx6ny3-a4^gJQ$IHb0duhbmRIW^eYZ1*HGZL zB+(uLaoVTvcaXhYTn?L&D6aBZFFaF=`6gtxrbnWGHZDXPm{SV2Sa`dsFB(xkIR01V zueT{5Z)M+F0spE;nDMqheNDDn2es~s%7p0e@*zo>5X602x7B__;(Z6JVxz~NmElZD zh>xt0LoQCkogq)(Y}ZRWTX(3mN{V| z`g7+E`p~t?o*gVh`TcJ8{M@im{plx+#;{(tWs~wJRR4r+jRR6Cnsjuzl=4 zTW6=4#fH59zA2uWJ3DR4X=M)ltOWN$9qbRo}^%^k!N?crfirHE~l=EbPcfeDb1 zt(hR;SfWYq?r4Pb@b%eCXI;()WoM{+eqKgj`MmaqZ>BZ5!<939azq+F{7+1dAvQ{r zqak(g!Mu!|xrI&c-n6T(DUkMg=gJvzz&BTL#*;?TPrLBE#|-%u601(8DNGiI>1`31 z%|W&Z!VWtzKAP1c(7rIa>cwmk!F3_|^A=^FEdqTb^M=0~jW!satPu`dfHgwTEv#O1 zgfu$-wwR>c7alD+)`|a^c#Yi;*i+=@Q7J4j>~7}l9)Bk-{4`(3_Fm9Q{qkw}2()|5 zNm#Sse8kzAyjHONkz7*Us2So+ea zb>ZT;JiRR^ds|)Iw(K06J}++ZLhE>jtN7}}jRjlQCOaJi4rlV(Ed@=V+F9(%=r+bB z6h7M1e5$;>EG*RJD!+ej&!dG2afk}d@+UrGXZ2(ov3kPr8qhk!N~r}Ca9YdQ*=w$u z_=u85GT|ubGs@}H%du|Q)?wMiTp{{6ut|#2bwnOa@O5ZqgmClmbR!Np>NOAtp`jT` zHcVXD=Ltu(S;DORyxg3`1R1KsXi$gfLT6We;1q+(5ae^ZAl)PZXT)hlbTg2nKd0VN zTW3#fzHG}?DUxfG^J9BDZJS;!Bl<y>qbP@u4l|g>bf4n) zGHOF>A2=!bMnf{{lu#YvZMJtx9ghFv{VY}cDaTIf1IHKeIjp?9gM?WW*p9(5}!B_vGpz`D1a?mZXonzi|;cekwfBX7QU-`QXf(JcUDI@y> z%CU+O-E@fUuJz}0yq_&vKhfW5%1`M6%c-Ag{fC0esh_+Y{nYA1h#%zbzp1t_t>W!R zpgb#}{FGXL;RBQ(k*R&P{zF>%f1-SkOzrb>bG_CM&g!}qGSLAcIG`PQTlj^y|2hY6 zze{_s{%Olk=>p5?xoZ7~g39T+yd3WpSf8HD%kkV-c>7$wTF2ERHPcl*wuYswk07736#|chvtS-*Oej5S#H^mul6!=MOQ-C3ihje zlwTm0*iq%Zd+uSW$NLld`x9g*mEEhv4|XPWZR#9c(WPu3KP9bX-DrOew(lI+XUR`E zms?}W*HvBaB^$<6CRn3?I{aebsF%%Iy!dcVhzUO42vy# z6zH^NgnHj_9lFLPZGL*;hp{2O7x+Z(B{SWI0}#xXq?q*Gr3MJOx5x(~f9grG>tN=tl{IB@g%jay#1Ze{z;3sxpdCQDRR zUHt-+MT#HR-FAGXYe(0euf7&><4)_&^TSFm5qNzjF%$~FDkF< z@?M2af3{`$V+Wsl9j}WSs3YB7DRloK2_PiGED1*1%t(2c1%0&$SDORSgW*(e6d9Sn zB(m8;$&GXh-01DI*}|BjR>#njr`)=L+X5<+(dAP0xKQJW5 zR<7w+Xdl#*{sQse)POfIw?wAWZU~toRhKbdcJ0vdTiCMa?=0TD+S6VdkslV}itZbo z<$x+`pK>U5us>%*lsLFA<<&DMhFv$M-TJ|eF=dOYN*fH)_vDayyN)%NT$ew5=xevk z)){FI?!KUxt_Qtph55ohPeeKphRw6$DS!%bfJU}7ydT6r=nyrWAyDnOmt2Td+R0$m z?8%ihnxUcwTqWD0C_p^0Ga(Q5{woBS*>PW1mZABf}{lH2ApwqT$b{ku2sh) zI*6;rDF{_0CP>^@JBw?i*!$}r&0GULk)??UOLui9 z6XFnZvWMRPu^#D?msO*HsNY~?15i?q1Ev*(nL`I3!oGD7UoN< z@c2`?S&;ka_(rUiD_})O92c4+SlY>_l&e)ZDgpuj@cbd@_DHv=;WD!)-+_ppdYP|D zlj>t`O={d<75agIrm-kRGNEp^t1KrwDIvt9mxWeDFVl?IC4RL{p+ur+F-fEMmBSG-q7<#PM^#_@ z!u4MSfQdSX;fQx$COF;X5HL=pc(dr5^>|DZW+@09$X-KBdO< zjA>>jLjyCL7BJadGrI!Vt8At-%0T?YPQ9dS%Vv5uAK)K~1v*PGB8rkxhXs*5fw^Ih z?9Sw_V}idYyzK?qXojEIq>V7%Q;S<+7t+IiN^zL3g+=R18o=(cYWcFh1RM=ADfSn6 zX{x&_Ha@53byq=d_Z0t2RJG1n9&v(#uf9ZQ=|a?-9=okNVJFFDA}yi#J8{M8~x_Q*x1CkHf==2qT|XpN55|BNa-KUkI!GWG&)y` z2=7qt?1%^tkLY0g&VHk5w63co*_Phlr<}d|rc|dWRYvSNAb#&b%2^-Ju{1FPd13lr#<5;H?Kt?I#N@RdQAe@b9({Xnvg2>} zg(vhsvU}gTfw<7eX}N2bO5`xP4LY+@;iM%FK-P)D-k7&JpQ|wCE8~*yXlnA;hcQj$Z~Vh>8LTwfKZMYZVClnigqr z!chvo%J6J1j*5;Hdp({vuD|~E`ug{3pDlU3X5zHHYNN?g#onA1X528_CY2PJ+1b#d zc;#+35?>Um^qLD$@|cpyex-EdGvbVVIizQ7tS3Z47)=O&be8%nu!`T~*upXCM<}8) zLHD?Ly?h)ev3h9>m0>qKDcp=$G7VUdpq~!4h`Lq+L(fB?2vYRwq_vz%A`uH45A9nB zWoYFQgCmT3L{3OGWVr!sGOva;O%h?da^ zEd!+>_sqgR2I1AtNX(!^k?Nm?s9=JG7$O;i2X`DIkGr0G8?G9Pt4TR<3;E$adpxCX zxmdFN1~}lk*lsxB&92(3{PgaUYfjI8dx+(Ii96wCroB{IFkG=xdG%@To!3<{!gfke zbdZDA?w=_C37@>vhkw)cni8d4$S=iP2omdMt@^XTJ|+tV@DnZb6lcX4v9B}IZbL>u z2tfE`$yopT@COm=z}Yf{+KkK$vKVvDhMa^WOr}5dZv|m?q4cId>Oq-eAYwnxfclSH z*6_JPf4247_NCWvudTi6x~1*cwk|x<-ge_&SNWcs+uDyT+^}vz)%KCR{J~9)Emu$M z%NyBNyjn%PTer~=~2h6{^S8-F59C*XTV!DJBu&7i~jU@-8I8g!oaT;w_(h2b8g{-?SJ z1;m(D17i8(%u=K+@m*z%ZQ0MZD#ymp>?gW=`?MPzXciD_i$~ifKcrS{=Rt^bYqY3G zlv#w%LgcQ1a9fDpMQwk?j_k)j{Z1bmvHsO?{_CeE6wbw&C0VZ>eb!s)!v8*BMW0WdO2YwyIwZNDcAJ> zuwOZAP8H2PJ!TQX6M9j=u8ZmE36tW(C;lt~eyEE;_C#T)QmG@GOQ~>N)!T$+GXsu9 zIs@#7Jcf5*0R#>R;iTS4g(?TMGU(u-MurgFfdD(;EJ8Sl;%ZWa zRuwbI0+?>kh{;F<{S1DdrD@+yN9=RZ(qIAoS9cze|URtS%asfxNYqh zZkH?*&)haEvufTP@%@R~flh11e9T3m^1|!zXFQEnw~DOtDL5=9V_rBorXK4ulu{5~ zK`uxVB*J2$A)#vsAB2V3*_k;M;}T(<9bpjuNjIP_E74?KP|pa4EW&yuNd{V6{4s;2 zcS18;wm-aM-&3mwZ$4UF*FV$}2H~XVrgm3DLq&Jt@dbyTx~eYb)u^hTh6Ov?Gefev znmc!Nl}arIXTI~wBO_OTq36NxZ_SUdb2sd+8TtI$@T8daj@*^^JUsU2SWj{P(N8xg zuNrRL^wdwm7Lb`bAvM9rOw=23>c;nEyuh3CL5P{Zq;G}5mwEdQ$CZU_T+C_KwjmuXJCMdtF-oB`+F1jf% zExNKit*5EhuCFdy*CYNSZMMF=sb?Qc&M#cR+z&mx_?p>bqmsA&(c#61o2%z1^?iPf ziFs~Y;Z=)MqS{6$-fLP_e$A)&J7+7y(hc%|0c#TOv7|6yonex~$p=BD!$iO+VEK{2 zFCsdp&&JyDMxo!ci!Yb{k6TlPYU~%7iDC&TJ}}w{A|!Qb%oyv8;j0 zCD9Tl8;xn6HEqQMjx>whX-$>lO7)qJgqyzc=7{p)e=2W%x+y%wY_FYfPm|+IjG6Y# zpR%&Ezj>1-Ji0jCWU{BsAsN9fZWd30aZM@hkK)OfY)xxj&1>R1PaWRtEDKp-OI~!KeqT>~gJI$^vHJ(Rwx+m>)7#3W zh?cy(B@5E3TxD@m>F!WzgC)A$leD%=IUA*ue^B%08}aU0NtvDmib7k0cYF-*XaU+W zQONh?1oEE5qmKah5fJS$!)djrr|Cn92Wf^ujY9jnM9VEl_tDyd{)&%1dfRP}eg4i> zx3uaej)_~3^!5)d$y!h~(5rKQ>PzR29DcI<)`?%%u$x!kd++Lot6ENCGJsED@d2Nl z#vGtH9CY^mK;91wPrcACED>H+bxkoT5zsst5GrvllTBp>h(S>f*cfR%ro9CGDy_Ao zrpm(#14t*GmN6_Q8JiGfGr(A2u^76yH=@5Vk) zYGVTv7B5=R(a_%5Ug<6=%FoSkCdNlu!z@BQs}EHbQe44F4o;ciNJC_?OAmSPlg~&=ahQl`;y4uYxxsxer~y6eIC%ZGi66{gTT&lxYW8Gj?tOOI(1wce zh~~`;_MB+aO`Ojk8y(p1P8>Si)pzrXlA5 K|DZ?>ggYV;{0D8ErN57pKMEyKQv; zfGaXLa`ig+N5wedAC)^R+EJOm<7=C&Zsnu=m(X}L=3n`9Ht552gv|IfLJTi&0rnCB^cL(5s3M@ zd4UMTESwj8FhmH@Q2#>Yq}PsgQ0|ByL|_$9ZfBuA_g7C` z<;+zvi$|3KX&>fdh>#-;Kb3$$Ox!J%L!mHLp_7ykz)paF*MbJ#H_||5d>2ohTn&qc zPF(G)4djEz%a9&wHwQY|E~1 zZRO!fJvS{sd2@G)G^<_Op+z12WN-06&C<_qUO6_>689VB&B$fP*W7y3Yq|>bKotH% z8DQ`7_j3r1D#R-W9VU?a0bdt=5=pf05_BWJ%Y@XiPZLvDilW}T?PcsuFJN6X&aJL) zF|@6u?oqm=)=xTno11&3%~*rF*dg5+X#hk`Qn3aw2|`XFJvOD+oS^+}Z*e7xA_M8u2K!q$dX${Eu12SefqR zTIJSSrFnnqN9-8uI3o{yd`23ntJyvNH{HwB{)I{3P3X2A+*>CG@v8EbUQeV_a#fPDOS2evR~rAv>IYi4Z^XM&9AGEn06z|1E)z05X=Pb)7C1Gqw%AZi zf!0i%kcr~t7;|+}Vwy<%Ip4yuhf_du7vN46gzmr&4)Y)O_GvEY*W}zkuyD;lo!HeEz29naZWq%s#ywnigFUliW7zXx!`~gWg$c%M9LuD^VE>RE^49@2zK-4x;*%f^L2jpfT!>hJED$7d(Hp9Vmv9lox2hfS+9k$# zqLH8y9tOw;NTXl~HH3zk(T?7(N2-ie9MZ}pyY>g32T!rq;R_d7jQ7K{YuQ&7n__3) z59kj1la2m#P3aHF>+O%)lU421FOED94=f=ajc2<4Ak83i+ND(W57d9H%2Vi%@>}hL z^o}xjtuhCD!Jm8|Xb=A0g%1p6Jl9AxR&(sJP?6xgF$A0;-zQ-d2gh^NX~2Rd&+yq$ z)=^d2*@1i{6Fr@kvlme8YWl=;lyqu2O1j*$J9(}W3bf60mSEgeTn^!8(lO!S{p`i% z1mI@WI)t0yb%2{`Q86(AbyVC8)q&Cj!rR;I#^5>_1Jp4*hqx<+LcM6E2&zzhFmniT zI4KwB2WSN~<)RVFMFgLf#UXJ22qXd+N2k>q2PieC zH0YT#r7cmg$fAV|AY{bnT27*YuWhYkuq0}OLi#)U&l6Q!jT~bI6bg|mIqjKFuac8? zV;*D)CaS1t`2IVw+Nn`8r^FOxJl>N3ENXaed~5TId%vUn{m4OP`tH6Lo29mZSfh@0 z>l|vVQA>>E+%(=@ec#yTW1Yz<3yzL%zI8#$*5)<}wDld<{H?Y*Tc!&?%6cgF*7Apb zvS;s4&kn74_$MF}m!NBh= z7vM8Y!VGkTS&-Qv0B5R_98{iOSK{c~f3RuukM213)y#@#M!xivIB{3qZTl9FRd~*A z9=LH;TIQ=MvcGAcvKKLSB1)K!gW>{w19- zPP)O*9mNfF4-nXwyvHc~6IA8*U`>&>N>2GPAqqY(iJmyD-dIKZXc1>a*jWU&Fry$N zlY8B8$v(2II6zC|mN~{@QlaEb?5q(FCO5e|`zzX89m?lwM)9?A)`F$ z+E|cQK7W00>)sU&wy>J}mYXAud70(&*7VNbH`HLmdbbaK>qOro(YLBECz$jSrM#l} z5O9UkI>=U%2xF(N3ak%*um{jf0MhYPMqx&t>MsPKGc+(5QkSz>I~~fD)G&(!HT;EA zCv~u`c9V{J-!Y)@{G)9mi-)qTi27g2k1d5GOmnCWU>+t<2xsL>XYwsc!lNVK6i z&DBD>BG8rX<8-C*<3z=LXG6FIl4kP!S{N0fcERZ1V5&rIT){1z1 z{p|K--gj-fx7QeDC~~@5=ujX?&{dy-oh=V%j&OMEs^@bQ!=)60Ird=~d7uR$9hq(j z3!iQwGL*tIcJk=~T$LJln$YcSDJyHCANB`Q?JAMp@6x&QxpT{LO=;S%>?~_;F2i@~ zi%4imz{OmU$xTA4Cy6J^l1VHfX+Q-SqaMtnHVHDINpv!1H;LjJc1l@!P+7(9Z(xJK z(SKU+8vk(oU0645AJ>ZK#8VS1aU1q`j;-go$pI7)?iQdED#vp_Jh^7l4 z-*!;CRX*|Ydh~&4)x}zLXK?;VgN!l9(-2Fi-?Rj42u{C7h)p>DB7Z947PZ2`6s(3f zm=e9`nBgx$!iM+>WZ7{-yn00oS19pz`?Gp(P5053J4fGceIu_Vzc9Z0Xiwkqo+Zb7mmFQhj?a6&xTLt`^?9$)x~^sJ zZ561TUpRoO6=N;)ZkWYPi;peo9_#Nt*1hC7-Vvb(@#{ahoLc!bq$A8quY^hnBN9(F zaoPWs%gRJOl94H$oL4yBHd!V{`$=g3y8-RfR8u8#K4>u4s)MVJA^*x{-@x|ms<8X1 zsUj|J#yC}B-n;-@O@+|Fnmyqd3>^ZYNF`2~D!lNwVt$+pV<91yAqXOvROHT$YF^I; za-_v8Ee|zAf286sNl?Ol7V*fH-ItI13i*A41$XK!A*%EE#h*r#wnV*47b0Pj9q`?K zu^X@S9*+8iSd=geN2c0pcf+g(5DsfSLS{Iu$%vdmzQ1KI_eGAYNDCKis!_i&uO<=m zxGFG)M&JpSef6I1Sle191r?Migxg@}G|hqP`|1m;5qy}<8b40HUGS(P55e~kiG#lB z9_LX2;5|=sndZSum^WLz=K+OSY9Zce!lzDJM zPa>iwh3a0&PL%w*1C7AwrvVu7ze#}@@K|uQQyBGRdy(LYvA}`@uauM!7ZVMIn&icV zr>4Y5=_G8nbdbTePB?H}h9`;|>U9;xc8?1v$8r;>AL@H531D$9lS~H|4*-uFvQ+GN z=insl_~{J^qoWDzjUYhzjVpj5p9W99Re6l{utZTIeYKD42BQ#yApeOBAqr#!+Q5#EQKR-nlu#QQugDdY}S z8{qp!tDz=_WL%aQlBG&qP51U0BOwKGDFm{i@`%*QpP4VXe6k@u7e0EL5U4&m;5$9r zClz|1{y!iUYF(_ocX^1-SOB4LO}|eloD&jdEP_xtLbNha>{;XMIK%sMRL_SE;VzXk z%MqZ@!h$kY!_#?crs_}}hJ+zABn%!))rCsIxvHnOgY9Ct-G3hsubL8pghZVk^XAsq zO_R4eoLm}w3SNF%LdFuvPm{_B&gcb91bv=VFVG!9FHrjGqs`4n*R8*4&L1hIHCE8T zl-A;7Eovg`mhq>(DXsNi3B2o+x5AZYx>lA2U{iqp|Kg;A)hYZ4rEBJod7>5^;v$utW(}P0lK3mEI50(?^ZtV z&zQ7dl|=*UU?UJe!~3k(d1#tC@Q@QvaG93Z0kTeOC!jyLBCZH{)+ukDfM=0M%XSK5RM8bg|ot!g>MPp z6JGUv=Q~DGN@7T=E9`cOM$$!jInj~C=IRn0wxl?F ziqaHb$`ZF;U7D1g! zveK(2p@n3dar_rr$D|hRgSr+?=A%=Sc?9c~>PtzyHcRDwS@Wd~;5D@l@tRuap&9G= z+c`D44&()F_hJ@wb>RYH5E+4z!8XHva>?2l7=uU+!!gNmX}VBD+)yyx1%9Y~+JS&H z{uo5QfC>mfl;w||6%a6sf+c_^Se>WFHU(bzW%(FnlyZ($&mrE|!B|5fP$0wkkSsdx z2pK6vrlr}^5X~!IMWL&@L2Pser@9s5EbA6uj@~|8(b3bGQ!RJV$M(eyIUb#StpECw zlA4UA*U@ivECt;;@o~8n8iZu-8RPBeO8IVhIeu5-t}N&YaLvN|IIo8U#bBKxmk}}^ z)>kjrN4?)zRNUBDT+}G1(%+J%Cdj6s1N#7B8c0iXDIdZNHMEZALr4wNN<$2#4>NhntZfrV#Z@S`R!o`8H%FKWZ1i?XkFff`OOEB;7 zm#v1rZ@1Kuot|mWbw(I0NJj2dm3ck{5Uq9rA&xl!%n$w&?|8|Yd7!doxcuxFu5p%{ zhiu8+`x~z5N$~u~t}Q9$#aYXh@_C6>eeH$wo191?zuTf)Z;1v>ZUl!=!dDf5Y&55hFtzDYkqLkOC6!%>}r~Ab6ct_F1hX#ZztY-%tsW)$4 zy+01?T(y!bo)T^YRQ%S@Wdlxsv8%ucZ0QZ<$+u)n6WSb|eRF z8uXlHF9Wpc=h+JJ)XTQU)#Y>7#4Wi0&~9f*h@WfU-nBWYyg0qJ3~ZZ|+ufd4Rl%{E zoNdeMlGcK4V?{69R>#A040GHK7b_KyS}J%DK@5{zCNwJI`uO%fL{$Y(9`G$goI(A3 zS;$|9qc<-LgBticGviO+ybS1Rj3_txfu~$HD8u`_E^mQf3HQqoswD$tNy6Ly9}iDi zu%?wcumb%b4^JzL`MCK1eIAyqm(RoF59szN@9LiyGK3Q0YA%mkvf#ETOBtMxRf!ic z(O^+_njM%{N>$vw1j}P92G{4N^GI4;!1r@L5P379&jUDLdG}eXXQ;GcdGQ0Ey(+aR zq~Ds@v8!fBS8VMoTh}L*7NoY6=&o>G@7v zS$Sf*x{L|p_(|RF3wF?8X8D7xXa95Zhjcme$Dh8Px7-b^bMw*liH)&snRb)w+qQ=I zWgCWiG8%Q)*CZ7z+F!rmXn(9dUkNpYGr5MXvnL-KS-vwC=gSNKqWl{?mJT$8IAxyp z#y(O;P);#(s65NuN#(d66f=qI`1p0ToX3R9<9bCYK)1lP|jExqY145W0n<>kDM zU(z$s1}x^h4aY^=pt@)Su*Kjug8PA(PgB}RqIV7G$HSNC$0^;^ei(dhsQq|2s2_yQ z3+e~OjGEF9l^#FW>KaUs1jUu2a%!8(4+p#>wH-t!w0(H`whi}9?uU6~8hWVxctxwL zH$B=CNK2dG8;`7a4{#|c5pKsoB|9taZr~3lt z8!x};e7oSIfxqr)mWEhoG&j?i+$UOUoZwy`wnrBo_?`$x=7BtWNb4Tm*Mdr@?lW9zK(HA3V zeC}-`v5!afR5ysmG@qugD?g69zv6xSkq$;N?^@woj|`^2c;^kDPo-Kvpg9XFr`TS+ z9oS6*`|`Y2KT9o#mIwE7UeZlox8xJvwuPUvTl{^|``h66GJ|@ZV(Cq8!x-v)9^&nV zXlq=aSd6-Oo>&|-7n9Yx=+g>ap8h_yJ^g(_bu8XKae8YQNvv%pz#P>8mGdigKc)n0 zdLa)!igmS>&S*et0pIaVE;XW5%c)cXR2EVu8xD|`5s)#;w53-gCBu)?0F9W)lUW)J z(hzh58-2M7i*Tb60a>QG5&p>9eE=NzZbNLaOWpxstN#wDsVu6R%9c^dQ`SPqhY%C4 z@+@Sxl60{mG^q*uChC*d<=1x`PeH3R8H}t5yQ!@jy*#8 zh^#&{*$6+La%L20BE<0l0i%z!*9U+{qi{$F!UFKDQNPAwBBw=o z1j<$I{IhCWZs0+YNG<6`3#xzG3@u>?n~?)=&LO7n*-RQYW~#W5+Wv@n0MSeBTT$X05}yf6yQ zN`brzV;c^;Op~CfV{6MqjF;J!vC z!mFFSI5bNERS(dlO{s~Y@z=zF_zVR+ulp4m3J}|xU3SJhB})9wl8NXmAhq%j-OnmN zNreWfS-DMrQZDE8ZV?8A^`5nUy2=vp{}hdW@-Y;F!*mol%x6{sx5#WAd<|&P!5^a@ za2hOz3tQ(^xVpRM4YUq;YF#ZAEoCJ<1)xPwnMkjYC{l35Fv@wYBi90+>pBzKbv2cF zlw73zdFE7sbCe%@Qw6R*tGx5xwb#DKGR~f58T3hc=j_-gUt7QaYoCm1e=kZ^EKLFF z1SdW^{5w{D?i?%s-QmN(Q@(rdobuh@9p1I|JTu;P7c-vUy7j#B&%5qY{&^m8HGFa! z#cbuWC7n^!a^GIZ%dyu{_!){1_kq3v>stj@w=Q9ji7IzQQ7D^1@|NydG{2@SFWm^= zHd^JTxd2#)SAHqOUII*>d?3&}I@Dx=Xdxp6JKJE=OE z8pzgbvMZkhg+evZJ1acP5@_IExkkiPfQp!W8b$p94hYs9-oc0D(PI)-_Z~vJsb7SS z+8TIv3(ZY6ZMAJh)y37>nLhd!vO>+lXQm9DUizxAw5%{GfUb!D6zuRbMIzXGB(3s+ z;d{P6vgwB>_J3uTW97AnGQ9L#D1{AfE-BeOuzYiI$>!;ZST}1{-7S9{L++X3C80G9 zMM2a{fM2&S6AkIqiEn0uZ>kaM$&Bm-V3!C?{YH05Zn}}OZ^If!UNH60vVkYW18~@e zpg>teq#;Usp}9zzc+7xyi&-=nI3##T{VWWrSGa&jkjdK1YY1Y6w;2ZiG=D2s+Q2Y8 z5d#|l|D@K|FrlWkwzZE2o(;sPqtFCd1~oE) zJJaQwS=j3ut=ppI;XrndwL5FRS{(<`FRA9Z+Y}`21#$+sLb+!&C{;%%W$`#;gYdVz zA)~nZBd@H-$5bXied2=;)et+@OfM@K#3V#hp(W6f30XY?gGn5s7VBsMq>wy-?o;v` z1EQUYr_hJzo2 z(TgX(B#z9}N6Yi8F}R~1$kRR#@c0V}vPBL)sA4+`SePFaW)dFdfV#^mJc z+X^|;vA4wH%NkW=P~P2;>^oYqN7t-T&(3-Be}U(6A(<~?1L~q_uZ%&sgZ#{J!bY^r zMcfbYojT+Jm-N0%o~TO18_T08{!7V4AQA0Yl`6B&6%aN7Hhgd?;>~PcZiZ zesBsi7)A}`k3oDr3*st=0!-h+G?Kmlepea8998?97izI*x1%xuzLQhj7b4l`M$;EC@p7}`bMMpIT2Q<{h z3hlj)NKHc+LG0)LnmDPYXIL|EEKyU9f`|!XiJD2vrJ{OO_@1>q3$f;+i+MCMEyiJq zwN{e8m$T=u=lfSyw1uBYC_eb&>itJ&<*j5T@zuk-syE&DYG-T3_RgD5FNhJ}<5|h2 zyxg^`d}+yV=@V-xn;|FfvAPY##mfE8ra3K>X!OO)rcs)gMo=0`fb>wXRs?=t=Y8b{ zp2C#$P);9(kh%%JO#}i0Qvr{yQHQE}vS+RZC{dqQS3pXb07?QJhwyFxODKuA5dbR} z_sLthO}5%oS(uDC9|Yyn=F=2f1;1K)x)mUiY(pr$7CfvRjuHonM*;H#0mU&r%Ayf& z9FxX7J&^WN+*fw?@jcEG^GX%awIreL+qT9v<#SgbDlc))FWb2}MfXa<{5ejs43Mr3 zq0yzD)DFPACk&4}KqH;nLplLso^ zlGJJ@9#qjQt<6}Gh{>7pnNgA9VId|V7Wu)r$Tfuq zkcahpgy?en0B(#n;F#ne)1^4$PO{hYy0H0gKvSxYN!K||8|d>T+juyr z4cOX(`vHB`Z)a!&c6nZx?DD@2=m+ej!EL}MH&Z`gYuDO%RJhaM2HDBA^2fb&q%|?1 z{JY+APG4_*KYe}I>AnEk@bZh&_kyqg{>cB>=y)XEI?}vP!=Qe6YRuuMn;3h5Vge=7HDK5&D z4EjRgxN(4LOEw4icOw!pA`~JvUIc!yG|0Km&rP0aNNyjMF}? zvOxl_I|R7yY$t5TMiT;=PH#KD49P%>on$X|UrJJ##!!r{ams!fC_K;_M|@7UVU31- z9aU|^LFK*Pxuon4Dj$;*)v;9yg351aN42pS)OErpMKV{eFye`vB6a035&6w%C}RDH zdO^St1o)(2m^2ulzs@A+u3!2Br2ev`FWMhp@i__9;6v8_rOumZS<@mi? zPt_FvflI!M<{CgTWG@GrN3H>DX7_D#wchD>4RFj+T?6JyR=5WAUf;avuH|u&a1Ag* ze;eQ$0Qs>DF>Zk~=KakUcFgGUkQP9Dcmx%fs9nNeO(*LF z(1-TCK)Mjliqi#qRA4#btkm-Qz;fD`)c#>_Lw&w4sn47qXT{sV9=5|j=KeO+@_;s` zVob5Osr?Y%75?ULgX6FGyQ}4VZ*xqyw*f-Uv_*;fChbw=e?$>Y_M&ooijOi-llssK!Ag%ajNw5nIX%!5+xQu;LejylY@%83Y zeN&H-zT@;1jK#&5b&zV3-oWiCr@-n$d>ry5B4Q0$_yxprg3>@UgZK{C&=;g%e zu$&=x9OBa<3SzKU!0I~!Lnk7Iv4)75QEO9&f3+L4w&aQ}-VOFn;VPiMtMsdz9Z@Ab3K{&tqy-!jPNH5gmkb7y&6HmH}y z)RYh?I&f;I4vWYm9@Js(;YDT|xZXzD)79u`T!)%va9R;Oni55jawbVswKE1aoVSJuivoN zYw_FK-jU-l&H7>CnyS976^F=+F3v&zbh`Q2D&a*Z+Kg7FE8@H7Qh3pYDf{hdHxut@ zG1u!56=E^dsqTO=NntQ1Q_S@_h`C;phnVZ7-k9stUD9H%lP4j>OBK_GDC+4w;AE_hbZlWV5C1G4v(s(5nuU(_NkPpV`3v_65vvJv?Vsiv3$N zRU_xW^Gu@izxzxYgn#FmG&=w8Gim7looCXR{(H~7^2d2+f+w}6nhEXmTjo|b*EGAz z3iETaoT*x_hbmSTs*lREtC8F}6p6rS!MPq(vLWQZ(jFRO z)C={#R38-K1BpC%d`^mpPohSNYA(dUlhsHK`r&rX7E5@zAZ}O$B#j41PQ}$I#v|xr z`shp8NlNG;y7P~1TDpH$(h6@vkAsa`3XoDyxOt8zCj%)!R%~=dM9kTe+t!_?jAgX5 zuTjE}P9*+_-SPENtD8mej2=-{T1Jn1w{1MIv?4hFM`e_KcK%TR&8@XHo9D1CISU4B z=Bz6!4%#mok&0-8XCxfMMHw~4CJa|9hN}`*LnPqTh|3pB#0nMHc?X3EO7R5(2#XK{ z#8dOvG>{$3-X6MeHWK;a? z^-J!4{Jz*FV~;Pn`Cs>U-`w4Lpu{;mRNruI&6WoW3)_a<_N-ZX?X8P$>KgdMt=+%) z%0m>FoageC`w^!v2C;LNdBPB{LW+q9HDOwS=~dNPBqEbbM@WTZqP-WcPqm?*}Es%J4Q?Nz{FqAzuaq1FtDAyURq258Lc881E>*tUD}@_6l)tyLiecuGjp5bGBZm%3 zR+m0P{>9G>R=vv=E*U4xp+?EX+^j+QIvxcfSl5p^NA%u4mEII#D$+-4%r<|b3Dt{X zn!(_~rKTz8v894W0a1p0qhJ7%f=4IykAkjWS+@qd=#9$p^(MoJa%KhFe*@dEm&3c1 zraO*x!jFcDiWF+>?2M4^7~dae5G`yITdI5(V}LzpaxB2p-dKPEarJl|;{^7lblgP# zvTw05uFH5!N)c>6zcX@hlv3a@=55y)qTYkLXN6a}&PTkAc9EyzvfF)l#&*9jxxbX# z;u7s{@%8r+Zyt=v{WaxJZWsEjOZB(k*Y2aj->1?+8Ik*W{6kJ(=my^+os~*w&2Mx} z^TSieE8ExJF!t&^@ZBRX5TBSd55y+|=YfCjc<;IN z7*Bo9Dc$fTp5q^L?Kzh*<_mnZeC;yETx<6-#(c+QI?+AY#)J7qT&K~A&jD>*xqeuk z1N?c@&VfqEg9pHm2Gi^E;Q(It#|Qn(OykaVn;1i`(*!T8zMd?2jv@9ZwU0XW-gYMQ zg0ai{j%ppXJ(b6_y5wjYv?drbz4R7-DppSa?0|NYiMPndmDlC8U zWFJ%$m`&$fI`bwhMm{sPb;$2VD*gH5o3ifx9iN8x=x~Y8d-PKKG|`vlro3OHuR&d( zsC?qSpz>Z?w|M>CLFHrmTCKbwsQh;JCr}J2$dOyRUmnM&1R+`2>9Hk2qCu?gP(7Du zApOE1m@;{nUExdT^zZJniHV8HiOHb8LyuiH8rHvzi^w~trSA@nmteoWsMIr}AFlk@ zy{9iB{OE*D%27i)WVkpX1^V_jVUuS=a)KH1svKtvM@)1Q z7?DcF>#myk$BrX9yQy1wVFBBHfNj)=*k>uvRlVBu{aP0KBjp=E*);c-*O(Q)Vb<4f znY-yH$~S(548qS0R9*eWKizT1pT2l?)xa|>YMzlr8Q76Uih<1-)pYsWLn*Z+RguAM(-Q4b8gasI&-D;_-m z#?S-G?|-}#a|^Qah4(PG2D$9E#}lT?$xMjjCkAa+Q3N(X7{ytvP0dUrjNF#+AA`hJj*hoRU@BI z7PH&U>;o=Gv$t=MqeFJH`*teMlS+mmGKYD;q4G^S}5Q?AaM<_KR_sQg5 zw>(^T8qS-7Y>?!2{`wT1o2!vQP(tE+BJ%U|3-b$cHn&5~PT7)~L*(g&vjPta_6nLi zHUm%Qq`DKrreVvFB;%U9lwY5}<9A=48+3Mfu$ei3{VGd5@u$vn$L@Gy!Gd$g@A%?^ zPhLO&viSJR%2%FW@y8$fj}uDAH~;#A^6Ys^+(Zf5&nSO*2%o$8@d9fK6xTlkO>+6# zJ?60&<%1@<&g6s`YnVwisYfMRRN~A=a1#aj1aZiywW6ITYEx65&;lQaQiFUXu%55R zj2Bf&KB}-VFOSyvyn?&}81o#E13fg=l7)nCk2CPvA;eZSPEfY3}n2^()&Y1iIPMLaqVVyiX z_vGrr;TdYfgQuV%CZ?dMpeQSoT;FH$;=Q$ybzpVRL@4s}C*`>UR)l0>5%f)|J3ja=r{RVRn? zT)_Z0f&=4*KtEJXi8`ByBBj$zKfV6DfC6yNX zB)DWuM;Q3E2$XDAWixH?y;Z#wfoqIfD`<9jYY|^*6|nc)kxn-|IVn2IZnK04NbhF= z>|IsAQU)sbv@<0V#{;k#@sU)$WNUT$Z{j-A-w2*HUuE0XS#z~w%Uw{PF^mA#W9OB; zxrMnciT!bV21QrEBpQ`I;FG8~>f;}7JyD;LqbpP81-J?5Hh`C+!Ar@)5lgH&+TCI^$_Gt1I z%3v&LiSh1<1o5)sM_R1>KzU1NHx4OJEoay6W!LDW zc{CW3IN8}Fvk*v6j_!cQ&I{|XaTPMWJ&GNUhzOp%NptytP|dZ;a+h+l_XpPvf9b}K zj(dN9-;xua8=9TgVdZ4`ZH-61HoX3`y-UCJWZyNLzWK?m$C&xY*R&Mvbl;);NoiAF zNJtbzo{&}+cR#Fr@W_dej#8v{QK-jzHuCo@gkMWU5{?PL+gOXu18@_bquueN(o^IlMmq$d~%*RiBUtwU@nB`TLKrnOBIH zWp?%?t4T;=Ay15-N~y_un8iJO;xD%h(peJ<5SKg%d;A!Gcj%OLPwM9(^KymoU!F)8 z!YN7^XxW5}8xaT`DVPbxM7Gx1*g%l$b(;`kG*e1iE!$}*v*4IzVFBq*F`>-}#cne3 z9r|VyEfo~ow<@3oo`LpCn($PscN-2+2d?yXtTHj><>eKW&!Vs(KNso5(yTGi+J}WA zTn(g2Y|vP0^_pzy^?P+@QOeb5c-kP<6~1ITkqw-G%K)_H$%)a1Fo*KJI}HX?oAPlp zdr2Q*su!Q{u*o(>Icp4OAq@>7EW&V>l^7zJWB$YsmA^uFE;F0)XFoHCGic9E;zOng zrZ0PXb&T?TF~ZQTeAr_MQy#bK_ncDNp|2BNqTXg!hf+Yl!jYVE7VXdM3<_sCd;4W-h5f&<% zOhbH#$Pf=tXW#v@5wyON258ejgeq!42Ty0rmMd+6P7fYm3v!4tghG`W4IxIuwf?RM zA+japBCqn_Y+~5Nv-48!FTo|aFkSMpvobRru$*AhSTnR7bis3kL_C@!g`#hYU>Qah zH5V<&j2j}1>tS(|O#{*cSH(-A%9no{5hjbn!=fA(@l$r4Y+;F`!e>0mxtSS}P{5@mIbb7X$w()OeNJj4)H#$Z zNS_vpF^hyPQ9J^NM(l2Il7$$k$&VC0I1W+-5_-eHNWz71Ed+Lmncf;Sb`;ojl+`NQKL>ukKvYMHNYwl=*4|3O9*_ijsiq-YX zM&95#;Sp>%56+6UGe*y z2gbS#a_x`zy)?S<%O@M}`8j)~wJSrei(Hmbm9cc;#N2{p3udoViV$DYpG4N9L;?Pu z28fqtC`_VZuY;8?kW!Q2cnJC=3kw=EoHlEkhQ`Be!Wpg1sv1^`v`4(KN5 z&>R6)_c!brO`rV?RK2{8@|rGBrZ|wOgLrrGq>-h2;p-4s-l?O3mQY?mSvLlrn{M}gXT~@c-q1R#Qce*mBrM>jL z^6}DLjnQW+YDU)WQO3WoynQb-b~*3wzy7IZKRk11+PdV}(%7fYzyIOl{Mu5}v5E%v zkFWQg{QIqPSJQ%t7pzsEdh?;@Uy2Hu0CAu;mq0r~w-q$}f7p8u=(wtLZTPh5MHfCBXOl?)uhm7G9R-oIQKL`|ULuFwEhO2dx$e;duBdv(T-8Y+x$z z$b)X7f&QokIJ6wv1n5_q4LVUY!hK*8(FM|EV$8jcR4;1Wc=3mM-ZW2r#l%m+{)jL+a*774Xi{AAbN_d8AUYs=-^ zc>Z==zQEu?I7KhahI0`tiq}ug*XG_`=ujzQNs>H7?`a_k^^Y|9M=Ry5i>6De~(p?pN`~0lYzE z4|wz0Z-6)R0jESRz*TU+2wF3t7r1z%p@=sbLy)TEbZ9836Zc|Hy$A-+MkGN?Gf3Pj zWR`;vjYdNcpoLmylS%2Mr41Q)Wstvmc&bF*!C0@yd^7lYUt*JAN#d3xy~*dv&dNwn zOTnR($xRkAlbN#2lG`GtHlaZ5Vq_}Z7jvkSAYO!IHDct%r~cI8h*sc$sZu;(p9Nd` zU=R*?N%VuqlSN4QnFT?|_kiWFfxe>Byq3hEg= zK7W$mqmJYza8y)5pHgF|@dLlVywN$2tUlKe&2oF7MlPY+{-U_g5L!uXy4m26w4x#fb?g?d2t9 zQzl+{@1hL^_c!i+@dbkWlLU9rzPwWU3HI)U#Hi%n3noKWNNx`4)5T~lr8NO>%n*1Q z;A5)FFGrBQPzeZOm$4~Ugu`&6s`45ias0C3#5QbT#Z2}`2w?SiOTUx$48OxK}48-^L3!9OoT>mH!rt6$lXDz@rC&M{95wHh4>}2*&Ht5sx>a zxK$u*3h`lR!o&MB)OalTK{#BY*^8Y1We+2 zCg2MiwvevB3CCk6V>+Iwus%I}_ENVF!M`4f&_%(okZmxE7sylygpCP69SBYX(qgVZ zx+i-jhG7^w$#L%Nv_QHKCrg3OS86~MvfNgdt=j!Aa8XqWhhZUkD8-Wv%hlORoadkC zuY7)ED!R_dH#id&m=chsbKhgC;5g0i<>$}GCuN!C$t1$B0E`zu_G?7}6bOGYL6j4w%wc-qv@+&o)2bvS&f}o~E(*`-&$1FW3I*=vV%-|F&Q88MilQXV-4*7{BBC ziIpqbj{oWS))%*JdvWWwm$q(wiJg56=7rtGpXOY!9#C(U#tuKKBY9D8U z?Hv?*srN3`3~b(EkF|L;-$ZTdZ$QgUsPQ+U7LUuzMZXpzWx_iz7(SFmnC&z^M$n+| z6@-q!QL82mt6XdW8AC!0{}@_{V|WgGSR(=d;OH+eb!R8UTg_UDN}qJ-`OPC!9d}X+ zzOK_UivlQ@DJ7zX6T44bXLXq8gXoGRx?w@;SU7%4eO^X}vu1Wn(du;-sm;YpZ+-aI zrPB}HG3By14(<5%s$lb;nd7##uDWIJ(c6yBU36OWdpHj52LH4%JfEL$FDWQlxODsJ z7x(ObVtZ}hYqwwZAGem*`j5Z7d(-`0FF(8L=54*bTRZ3N84DccIl|GC{I?)yw=4B( zCPZ*KuZICFEr_4^&`+oiVDs%bK|>k1)Eiw|#%n?6A~C`S>WGkrB@uIi(x-gPV5@%0 z;FO=6o*HK{8#N-_ew7NG7#)_vj{!Kb{OL?&m8h7*pu~t3u%w(50EwU9)YeduX16a#Ac?=K)85}(t zKhlD(0Tc^^C;FNH3$W4Ntb-eJFl-vG7!Ee*Ux1sw2L9Siu7KMSv}!4ojiym~$e0LO z@~{n%L4lAPF?1a=@K++51n*Bb3CTJQ(~n|30qbGjz>}PYtoH0Ijtls*3bG6EjcI;V z#M+F8wES}B)Q*ZemxPQ0kK647OIIXVDI{3=GFx1nZ65!f`NG{Re!`b* zUX``vP|B3bsanBq6h36m$f;W1VU+)^zw@@f1C6umd#aZ0)yp&C^W_R)2Q1L^K!WCQ z9l>_W?TN#=snS3>EbQ}v!wxwLlfkaEoOC}{Nb(`p#m-?TFRIf24kf^eiJ?Q_49K;Q zI72~8Vqz_yJ1(i(x&8E>J6aFFJh9CYoD*N}K}v!o#Pcu3i(*NMD8}EW>FHiHtHN8G z-O#C#KLbz0{94=@+U`0yH}g1PCbCAzhVo48kUZl)WCLmW4Xpo{K>fea@(d)r(jXCG z878Ham8}CkixE6GW7pu@a2kr>Iz2kE(!WyC2-q~ z49_PCw*}sb{_E&7%KS3%fRrDi8>|fxBIL5*<}tPtAqi~42vh;( zY0ch|9%qfbvFp=(nGL<0_Sn-bRK|( zWn~Nxxi@?5@$vQ>QzYkn;fA#d@iRP6FdF$OU$SLYM)#rA$>j=-d~8Te-T&v|X(Uyy z5MJR9z-Lg%9Sa(99^L`?5tdPLVc{Uyk$eclZpaap=H^i_x%dkJi7Ol2!k=B}%Eunk zL1lKY%LEHnC9_C~T!hwcY<3#WOKd`1E&t#+XO(mNO?&Pfdjte$Zv?@)6ES?qhVQv3acILMA;$3A5T1lA~JF0L$!z){+SK zXaK0EFn9QSNWWon2^?PhEu@jW7<_AaisV@MkC2@D$9#t_c{#DN@8&L>4=I%eamD9C zDk-FRWk&By?S@3e&p?iehW3|wGoI6*+FvlIMlWA3)%Sp(iI|cI|A;E9TPaVdT#IBk z64y<6!hpQ?1I{Lch@{sDk!{w;L5+pbj<;AvRcsdz{~!sx=vSEh_7@hC1TSnYY#rNN zS6ed%9Ri95eX-PpI61N9*28YXrbnQ{=3UI-C)rMpYIj;S8Ly1EAej&b?hvV`#XXE|Etd;zBsD0v_Ky|jCzT5E*G8Y zw*{^7(0R?!f05Z8w)VUTUTZb1t2d)k8KOp(_biE&W74N{4&D`q;Go7uPrw6P;@W@4tTD zb5+B_2RF*E-gYK?f~m&#-80kg`I>8bleW0Q{^nW!E4#ls&plT!q{m6EEbI8;o0sox zi*Gp2|MzA4wEh1Uu9d}O`?T8r--;sAOX#jkaMvFpzbu{6?+SvC2>g&lz=zRqDKwib zctaHLrWOVQw-nyvqm)O;yi7;Q8g?P22$8i-gC87t2HwrFp0QqMzE|m#t&(pHh=_#P zVSo($TllECZ)>G|tlf$~?R@W7k2lQ^)NShc&W{pHbUxF*zREXT)3ibRvGtN>Vs~XC z^0;gU;TnUD*Ho-r)G#T}=$*wEeMOSLFU*mDe5X!+Mw-oF(4g|LR9Kef-1^7yf|k6y z7tBd2{gBHY3G35C-#uDXoP2aMPwKzlUyt-&JVb=(){UO@?wM7jmVHG9}sb2 z9>$z&%!LiVwg>D>WF)Wn13V3jGBX$PFTQxbW=>)A)(JEApK9*7@6czQXJ&%6W}G?6 zeA|xgyPZW=N!4|q@>Me`Gb z<25jS(Mb`J12hK(4M9X~n`T$CRv_b9Yj;$>1x}$LW?&Ib+Q5HCE@QVjy=FGoFHBX) zYA9EIOHsTARa=a-oMB#*=t^`ke&dYfH|NeN9OoBHu8z%d_)_`RqI&zA4@PjEU#}1@ z#HAt6Kg@SNxon?6jE6rZ^Tc@`St0jBOD9-LTt4(H0lX)L3!u_#3^zSE#Rsl~&ilwp z(!!1+pdoE!CGr|L6r6CiM9$2pCVkB0zR|4yK|$c$(`EVv8cjPTQIq-bB26g z;wQDVw9Fhoc0N4xI5+tpzYCp-cy0ma;Jom-7UGCFP7ydxzIu9`kxq-gq9}}n&{B90 zfr68|0Sz@uM@$NtHae{i@@rJ@e2&Li_$}zl&CNt!+L+4H!rX$~g8V$MDdFcis%~Ndx?z#cVzDu1HtUA&h39rgGc$RYctihvksOV;n}iQd zc10oU3kuqjp2>69HzvOQMz-QEq$wNEK=7uqCHDgPPa;x)b7-PUW(_3wh06LsxT3r&{iPU7M zobXslu4GA;;0WPZsG?YiEHtV^ZzP-CmPX(CE=l z2uc}rG=&_%Lsk8Z@zCPI??5S0)VYY4LvJu9>Zqxy>8Ke$w%IeiRZ-UlvC&v8l!TP6 zM6oXXo;-n|NZJ~&!zYOd|KK935u?>)d@k*+|M%KoO?63>sjep>`%jj7Ik z%(S`wH=)b%a(Cx~U4_-FrcAkN7Qifg_BH_i0BR{KS7k?F)e60{nj&B6yRrodrDr=}bT zc(hTh30+kUb|B`*vN9KU4n%G=JTcG9HD#jEw5A$0TCmUJcFu za^~i6TuD(*Rc-}@pVwb*tAR2Y9TbYrZD4(^h^0#4r3el|7i6{qdErU{H;C)^_#ggW z$#S^YnONSQP9P+>?)->f_SSQNWnyDV3xlCF-kdJ#=Z2CVQZyRCa_yUsbzOTS0IJdS z|6EdRex?Iy`TruE;#%Yu2HR4uk-Ii%!!T*SPAllRk^&y(nIYu|E^?hlhcEzwD+nR# z$PEhhi;5aRugIZO+Uru3VHu(fkaz0ytQ!D$0K}d8V(4ia%EF$8f4sbm<7zQZvb-L5 zUFw|{fRSYanvtX*EP+A{&{K(jfkGAiS;vZ&5*0S6;$X{Q6N^aqzckg&32&ag=WAtK zZ7Gg*(}@3Nk4>=U$OOgoIIR$6WHI))S`vu?8ifnv<1or*uJrJ=2QJ$u3H|>DaRG*e zpEJP&2|pi?ji0^{eh!aB6oFK{$D=_x7O%v$43C@S%NxK^a(fPgqJ#XLF;G-V?2ig< zF;DR@IC}c;$2WZP6T;Aww+A1(XE+SKT%%d`@^#mopVKwz+TKU+0fufNdBS43(rmyp z8(tS!_i$H&-bEn+P8UK*ieP1e8WHA#^GSpSBSN? zK|l{_7krxmSFyVg-tgaTEN!^1(MT%lGW^(scC5fHes`nDkuK;dwv zSq`6n$%GTi`7fjMi<}+!G+Xm7_`DAujt3v7uqbJg6p;Ae5EAwKlM#}{Tw4t2&B{pf zC3~<{yLU)P^dj(LQBlC`Rj{-ez>)((H)Cp=+5PK7Xc7S?>4v^OfC{)meWZ-Q6(2m6 z9|nyS8%$;by0Cq8aFp2E;{_jnkEDkYkqnLG+A?!`> z?_2h_pX~e>NcLFakqCqhS~Hw+=lQmaqAczKRHVs*Zc>Q(tUn4`o$!*Pq*3cvGvcXd za1rA5f{|iraKuT>VNAXB5K8OAHJVWTxDu(e11@BR(u>u%jC)7MtvnE#7IlVm5gg|Yr^c)+A(FN z(QGyT3$WGULTZEA?Fajp|5Xt>&MVhmgUr{qjf^X?R>-!gK&==4rfs{(L>3 z7We5hOyo2F5reaidX9Zaz9(a|=Ahzx3iTWV1DN{RZ#5Ip6W5I_tyXSBFbyTgXcq}# zRFZ9ngDnW99pzG!V0)sDk1BMt!9K`j+zZwZ?cfULm6cF>DIFm7ipq+T@i}r<1P*#T zz5~iT^)Zp8ZRp|C(p*;)@TI0C*fdnZPeT~tP0@u*^H7+FRwF(|av|KHhEnh^c?;Z>dX#-G}E z&;D!W+F3mtb8?>L*G?|Vu;phoWaUrtSj3E~8M1!arV@)jxHzM~mFmNFT+3%3DP3!V zKO&E&3Uke{_B;5OnXey}&*ba3Fke5J{H-=qyoiQ52aOmt%k!wgWlsG~_P7MmqT_Uy zO;+B_apsL-<8g2-K5UpRTDDg&kX&@qj!!A|BJ|S0N=Gq>GQAoT1WqQD}h#bgPUQ@SSdiArT(Z%>0IX#YFGGol#`uU4o{c)Mvk`km~8ie1^yrXYj*Ni@^p?HyUJ|rW*gPhzk-1uNy ztn)$E!fk*&puy*a2NJ`2C7M1jX{LmZN)u;e9W`X=UXr6quEA&R8S$tpt}g+u*AT8( zMc{glPsMfOdhRghs%VaS6cvXtxEsnm#TK&xkC)kd8*FC`X0e=bA zm?wp=KR!WmmNHE5yfOx&PhS8&q@elh?=mERco34m3>?3MP<;0H9{_@<@L>#xc~KO` za2|5%9nC4|y*1q0U;?a9&hOL1TU1qE4BuSHeS-Qh$b125{ZOBXn9<5aqv@flB)$j0 zVQpKW2dEE$641~Awuc7Q)9ND_%!8IDsd;`!9;-U6w#9RM)C6Um1i=WDSF8|j$dH*+m*0uEKkIcGh@s(+U)0om#yXG4As@K2CAMqca zIQ9C?V-ua7vU`;?>9{bK@8ieUl*uVgmtDK8@7ufle!lpuIYnz(A~Cle>7!|b92mqJ8Gu-c4ImQb z-XP|{F`}VoBvjWtfuvl&TLTY7I(Ehu;QB6H(&Mc{v+om z7P-q3oCS^r2ihmyxN+Q-Et7n~L+_tD`-kl-d+QsL%#WDkRzJ0UQS38nLEX0ah1WcEcI9{>pId|IAq6Ouh-|+AX^H)FU7Y{RzrBZ(>>{%$Juq0}mZoaco>uz1M5it+E|oz5LpA6tQuo zs1In>tjC^ERq3z;-l2_&lny?MjZu&Ckv7&^xSn z(f@_66*z<(t~x z)-MP%czD05~(CJ!YgT&m(S*8cJGU zE*H0k*AFP`BW8X=s>SyS_{&N!%YeH3hqB1kaM za5Y{v|6b;drqYI_F-?;^JsB+}>-dt3F%)@6M*4JLS$Xr)RQtHawTJ+{K0J#dSdS6> z!URSm<=o9dj2@GO48%AvAB7`Hk_U!~0XixjOFEsj0q3Te1o8@SY^@HR1+?hVNcD#r zbY_G;KP@d-8u1F|P+T$cJLqQV%BGf!xq<9*S9xl(Bf)CcYq>16qQT%Bz;7^^2KW~0 zgQ+=;0$&DRW@z#&BY2rPD`l?2%L+FvcU0wO=friECN-ruj}wI$oUPp%_+^5v)L9>> zi??-F6cj0VStxw~ymZ6+J|Q^HnFs{Sg9H`OB!`oV0W#>8Ay|c=1o467VdT7ofdstL zOmlkuUN4HsQPJpA>YHO7FjSO@v}&68S519wU|Ne?!#HsG2@)#y;=qFakSQC+?c*O0 z4f@Nhe}3zr%TH6qi(F7Q|1W$~y}Yxf>Ka`&qq#yxb5@)UH9aVJlQgAPG&hd32!h($ z1qE(IU~r0Q#2ye93Y)cBp$BvZ4N6tF(9VSp7I3<#d)=UBD>j90#4tP2I?ck^50(3!71V;UWjP&?qb$Ed4{blaH3meT)y!6}L*4 zhxn&qwS+7cY`k60XLsf0FlCWpp*O7Kgz<04L2xqxuQ%XkrQ@pRf=&!ktcM)|NJi!z zu43#%s_GKPvibYR0n0k}w#dhQ4Ojl|6#o*Gm({)Xbq>p8w0*f|V23{$Z zAw?fjeTvl%=J-)KWskFf1-p4SHJa3bMa1ACArhl51NH#rJ+Pz)4<;rFX^P?&K7EMB zTG}3X^sakuo-;3}P z?Vot)gX`bOcw&A=PVd2rnui!}mJ6GOPRNr|Zi~`Ruqfhm0=E%{8ug|i z+gmv%8YnQ~L3~6jKz1O)5ebeFo<};jhZJ0|FJ5m!!ZQIS0;gkKxT5S<9!7en`jyqLFa`I7Q!<=K^9y*@$MT(l);`oTtD=aPpG^c^1; ztoM4%x;3^Ft5+Sb59HQO%W|(|wAo+I?zfg}2{yT1G46MS1Nn$Z>53_Rd*~HkjAAc# z$*@1dXbd8bzU2P$s7sDP)q^gY)s#zhz~eIR@Wh8xK)+RR5G!3rpcCk{o6u(*iHB$c z2ROgU=;xss5gDO+9`{`x{UZ6KNA)I6#dTMBypDh;-k?S4Q8lyIK?Ot< znL?6>O`p>0V6u|2LllC_BZawDiQT_6uKVeo`~G@*&3)gRI`Pa^^L4+}_#3B{H13(N z*A}p{8w!@@I3J5bm*#RswSfOA~dE+)MN&#BfOrX)B~#%LI&M1R3Ss89ct^L znrUD~Nr}XYRR^0kG@MM_dNr<-%*j%FVeSIr$s61EwX~gG35Lc0Q?VzBC8w-@dfTR_ z))nS&@i!fOpW&xsQ(}Lxk0CqItsEO9=U6Drokr>%_6f%U@maVIRne~6$3s5FvWgl^ z>%%@K7Ud!PSZ#Lz|5?N#bczxH!o=~Q$!vc8nw4kUTK2VXym8XF{R?sn?WOLbiRTXQ zfA?zs{Zk(t4Ena@7p{AHr|ttgO&_6WY9!dYmYcWTz$Z7_wi&6%n5maSh` zI4iGUL+i1-q=r5+IT;IxgUNYqT57w~|6tUfU=Q090AFNeV%`A3cPL<^L8ln%3gDP4 zVIbKg8frr8UB_BC6y(h+T(^GNywpr{wllMI+uT{F=1G+YAM_`+rKDfGeD(3THF~qh zTi-P9IKd0@SXOkrI8ZHgyaFie`Mjvf%+P6(PJ_54+V{aDPt6KR$7B9trU*socuTMn zGzr4i94F)|%{XAXX%f>-Q8qIasDsg0@Pt%UZ9xt{{UW%mjG?|Mlj*6VvrJ4L9=D^< zw{gY0O=i{fbPGAJ3o zFwg}wU!Wuf0D+yVngRC~aiuj9)8H8|s=f&6sxH9xmVM(keq~bo{sn^PwWHGdLl^21=nMOEE+#xHm0b59$Y? zEnq-^$}x!DnifGD|+6=fKA&LnX68rd;azn4}kr|JVQx) zS$TVVd0Bf(&^OFk1bf-Iar6{uznz;Q%@Fqi-bKNDbcqJNQx_G0U6_Lt@I5%7z)$13 zbf~6Qi?U@cMNpMUDQF&hsxA-F3#q0_9!0Mc+9NTaISIkleZhkti2E)=K%iFg7r!qq z0UiXnz0}ExK+LgDPUH@Sk)$fqs8d49GB855?-3QzN$78m^bnF!jQtPz0)~AW_HI-& zB2%K;RTYCN0{(y!3r6_><9DUhgjrlNegDM!ez*CG_ik-HJZoWs z-X1@$@#G^0ewTdEbl2TA?Mb3%Zbn+#>A&qi{r>)x)IVLGX4Gdq_1x^|zn@^XQy7Tu z1Lt1B&b{k#7e1g>vDk(d&gB@9A zi2l+_xXP253#b~sf|Wy40u7qhGEul<%kS=;aB#+amzdI4d-q9`e30K|yzOLDn=J|D z&~pxrz4iS){NMNg?Q~k&hqokH5?*?~_qj(h^v1Ny|CE}tAJU!9Onr)4AQOFDYp^++ z9eU)`M(gjR6od@kyU;S}`n4Tf>Jr8H0{@yW-A5m3mv`kBnyiwgzGhCr z>2p&3QD1*sr*PYC;9nHc?He_5% zyxwSsjHeC?)a9;Z+-8I zF#6cd2`@aKfe{rMZL?BSGoE`mLoYc^VMJoRp{2(VV+wF#8h$8_SPGIvS#Z~ zk2Y_gHr2t$YvbC>j+}q=OmQ8bsL7lx7V^nSyI$F_=Y_Q?N#9-%t5uhB{+nlCK<5$L zK^#rxiEo1^`nk3sde*}m!|pYFC^F%2p)nmRJm+?$r(z^Enh|q;-miF7yd<>PWhJ63 zx^h>ch!Sl(5sJsrr{w-4r+D21)30B#nb$g;Ih_S-A6owNXFW|>PBC|#uIKuc>;H22 zrVsXKr~gAYVWPq2=z8+VcYkR%8mxE!Tg*aiFLY0=?>X*?`krgjSS}PFG2@fnC7T(l zJaf+|p80UtXJ{rPTL*I%cMtt88HgC#4;L_=(CDJ}C4}|`eP^n?gV9qWvNqlhTK#Z0 zhQb+Scu^y6wbTe-Ks6vMFMt4|um;#dog01*ZmMhF-hVWkA|ev;&=JX>{yyz==L82u8Tm zS4|u>>^Shvk02w^{e!zYh^ApM9C<$(k#jOgEG0kkZ%*9Qv(QW zxJtp07ecMg4sr}*7b9bcK=`P!(?&vXZ^OZz;sQ7;Qx;mg&Qz_4?{-yfo= zT&Sd42qaP~z@{ji2(rRMJ-`frK(+Q(Qc*=9=Cc~GB@HUc)hA)I!U*>Syda)b)hVGZ zg$o~jFd&hJDiF3PASE%et92C=JYD3|#Z)Dtz_e|L^~~Mfz1o(-GRJXSeM~-O`_l=+t9q&~ctY8KvyI>H204V_LOQ(@A7RAgPpy|R8 zLxCsklyHzrMHbXgv2l1%i3h(LLYJXSP|gdz$S}y0pC1VDTz*kLhM)xU0(rSPilnz% zjCzvv7BzknrI&@FirgnfEdx~fYM0K3|AjZM`|T}LZ{qEb&$?yRid0Rqy{mcKv+Mrw zYD1@xuE|*@ExqN!eGbuVzwi%-Sn6;^t7+MB=G-VPjlojrShVQLF3QyvxpAMZmg?YE zqsr$J?mrbcqUxLJX>p*i3KV&w-N1%H6l6A-MEIwX7b&tdC{{|QfI`W~CF(4vcY&bM z3hW5421_sgmU0)XShf*CUMX<&6c*;@!jVx@SW;Z1Rz-U<(wq(k`W!xoK<{!FML}Lg zXl){;ol(4l#ui!8XSAICCw%{wzkYqzoxJDU^Y7ZQ9;sIP#FdSge{bt2Z`VvS8j{4k zHPWiD_us+4+9g+xXMb;e@#90^=PMq}@yS1a#F8$V{rf+AM*ge({S1x99``dDIUn8| zv|ssT75xwR3)~ziy?ManG-RQy4knN$=?8YYF-kRqx0)$MgSsiUhheDI87L}XF`CU5 zs>Wc^05mp3_i`h}#|@m(VlYPHw%Kf59LHO&Jyx>XXu{067^@jAW{}F~UXHGLeDrdL zy|lY8wBF!zMl2N~!Qr4~ru;^Qfgd64rY zj0(sM5F<+$S)(3N)&R2SP=_O&V2OD<89WsIBT1*d8$K7l>hmt z(Q0jNwZ`io2!5tGfyaL_tP}Zq6Awa59@QO}gkz`nmu@=Z~jGPQQ7HeP} zEAYc))|pMn6N+AoGt*M$i!T*y9%+@(4Tn|>ZwHx_3WJ>&bhHHP>uOM+;&s~`J}<0Y z3oescyZ$I^w+IRzW&$(9CbAB||A*h8)^s4*lm-PU69Qzkxh6;JWN&hHT(Oknjpn);g{dK?Yjvl;^IAf3d4aqj zThP=m(Mj=E{x{{dN0K{g(Q*m;eGK8)^sPf!edznpXLR^iBAGS4z?e01 zlyY()a+Dq+?l6}Q@BtbNI}DdrFO&H#z@&Aya+`2dlyKx7ddH%{((!?wjW7Q9p&#%S z4+OxJzX_&n%-#PP_l*2E`TH*3l&&!4*R?NyQU^r}c1%9AV{*84!8oEW_{kv)QFKnT zyZ&cXPQ`U$BaiOLE*qP~m2GVt(Pbm&^FKDmN8{yCr_F`mFx&03_oR0{HJqbNo(>L9+Qhs8| zR~Jp*G^K=pLD=<+t7d%tUw7R7m)jTRwC%WcPQli79X-D|H=)eq&aKQ9-j+YSykqNO z1|#O91HV}rG-Cb=ZGDXBi2TF29h&Ddh9XzTW>6R;)-1^AqhUniO1gAFaPQ99rH!E zrg(>D>k})^^2R5+gtPx4e+ynnzU664hQ^$Gh&R5(XY%7Sg}BsT^OImJZ2O^nLBWU6 zS^E&T5W^E9+=GufX#Xri6Bza(kW54r9Co9}>GL_f|??s$K?w@ z+~&_g3Hh&6E;#JPk(^dojM}-r=(|z|GwL5Fq(h)-6KzSIc#_RBRzZn2R9x5%k^|O z1N*&fhDqKq$N3RaYj8HDRfM@*pSG^ zPRUr6ny|nE0}4K}Q+|aUt)+^W_08u7ysR^y;>9DUi$O}h0d zzS*Dm1R)@seaz%$0Nu1MAIe#%++K$aW^fiQW=RR~WkG8|W2a$QO#YDPLnsh>PYowA zUSmQMFyl5XpU*KWZiD$JN}L$skqFKsIt@u{CWScwZWs6PPHnDu$n6tNakn?II|cO zfkbXBFzK@k9+m&~aNm=g*OiE#p6T!L8J7N6gc|+TzdAf?V+ugAdFPW|Z}H9F6dpMu z%dhVAdik+$;jnxGpV>oR^NksoyS8oqQ+g$>QI9&)B_xd&q_)SIv=h+Bw17YCy zki=6$uQ8y>k`ov_)!=I;O{}RZ9R-M!>|2`hNgY8jQpw{F_lQFAR%Z&@g%m;qPA*-SF{tPvtDex>J z&z&OWX{neHmjZr6eS?wibP$eK8rci0hl8qwxfWW?wLrL>X9*i*lWF;B-kbykCF5Ch zRZnGfwg#F(!j}L`G+<*^sS=zNe)d|y^168&t0vyKDSJX(@7eN z#}jp^|EW&04=6E0kdnyCNQz0LN1C98^xAH4sG z5Bq^f0qARN5-5(tNyw=QA0d=;0;>lTuqbyc^yI)ZbW8~dEMb5u(x8>&Qtp;|V&t;1 zqN>rITK?~s^A=anSY6hBVs%!Vwd>B3w5pfmf@zD^FP`H^f2+pK!kK%UCm)+<7Jud| zy6~?=QPap@C7JfD=-x?b{4Br!eQC087h;UKQ^Yerl(r9<(Mvh>li3_!_6%yMem?XW z*~oAqw9Y-!_D}9qX8nG0r#kDm9sNlOf7mm~u}I_oq3i+uNh5zhRobrnzW?p0-_M}m z!-Asy{8^*Sg)89xBk0I?Ab-dP>6o5IaWSw*^8Ha!ZaMD2Ac+Pv@G=-AOiocp0Si6Q z+@Z~(SroNIhSLVEBXAo$>P_M3BP|16`;=Gl65bXh@_s9&Cd0irx_97`8Tc|pG81xg zatd+^>|U=s0b(ZJ1g{J_uQ1yX#*Y$?SEk_t?R}AG&)drrMQO79Z2OsceK&L$ZTjHh z(#^><5I29W{A_Ap;pua?9&cRBH>W(*yS8fToNFJPbM%wj>{@<%>y&?=$rH7!geMJ& z?>=?UFN?(!{Nt_iHN>n?+d*(gbfB919tNAhE#?A8EXWu1Iy9q#BcBtfWP$)(ig+2K z^&%x}!mXml69FJ#H?>IvWc7LmZNwNOCoQ8%EE^e2JYh6%!BujR7XZ^iUjx9JRfwi} zlrSb^Bm~e>1Y0V>NJax!?>mvt{MEPUG2`R2vJynn)Nt7lad7RH^>uiCwT1bC}> zOmI$xMVZgN9&}RwU53U8wxnUUE@5&A&=p~BVnA-XL63fX%2ex7q4EIh4ZS@wDv#ow z!KiG_&r1lQF-;~OB^r|oF)|k4+;12DJOWgC&vIZH;V^i_e!$j^UTVJx+G$d85^t2e z9>IihOS~z>AgWM8h(rf!H;XZd7+P47QfiEOJQB;7UP8Y^LgG<=l=z~e)vQ?*SG%8mDpulg;mMrHZ z`GvIqJTZnSG?w4wbBSY|m#+#l3~Bm>QgDZ4@nY1rF*v)RotGk( z=tSkFK9x6Wp`;q~cnd|%Eri3QgH;TG3nJ2}(Hd7;G3i8OMs>VJ1IVkOuIuD176xic zFhO;OG8Gz7j{Ta27cEHtSKV#Os}>PDSE}1xQdE%d&UNPoe6H*SJHSPti(`||9U7&W%P0M6+4ngS!D6 z2@N|Z+}8C#=B?3%RE>JhXfPh+45Hy+tc6Bc197U)_d0H6U<+U@nH*PNTU9}nlAGi6 zq^Bl1!{gB7h8>44j!s39ejIZwdeI;*G(>077$Qfb|NFlqSy2r2h4LTU<@K5kxVJP! zUxzF^;v?THKd?0S3@6X;^Ww_4*M1w zLMb8V4q`<}DM8c^A!idpu?B<(H$ZkU9f@h0t2^~pEgH{h7RR9-G`gZFKXOYf=lPOH(QNF|5(M6MVZ>cflFo(RM%5=d|dPJOMS6b zdOIU70@+u}pI~8GS#EAdI?t6=mSJ91ZfS05NiiDSyVG+ra*~sr4qH4+=t$$!Y%G_C zLMu_$g|oaok*FFihN{DAgQnVr9}(^4k=Vh%8f|7I+nG9HYh0V7C=dK9AAg%QT}+%R zTlgD#<<%+CM#ug3)FmM0W5vlKHDdD;cii$%-);$qAh=TxNJl6tlQRh z_Y@SIM|FReb*}tO+ih`;>-{M!3U7Sk)&UT9?Ra71gv!D)`*GKVJ)15pw4g2XWPtNT zL5d2_UjTlnzYy2M590+FY!8k_k5+U_mju5Ls+JUQfj5F`?3f=0nQ@9A26F^hfh^Ca zC=)m}@Wc2p^Ey+DiH%f1QW`)@5T)bP5cdSDq>NHE#q}BKmlatY#+ug)`s(Iws+x3q zQ+7xErX7t<@@eLoS-v1=Jep=_m}kaMUf%zb4x?&{o0*?x<_%pYsb=k|f_^)kG?v2~ zdiSR)z6!z%D$!Q(M9$Y#15bRYCA>VCOx2f3Xc-*v!{-FGQ;X7P978o!Bm6dGH)%8> zyGe1w@L}^bNRhmdqiWd4$Y)asbKM`VKcu9X*cK(4Q=#pOGYUMK>Ync1cTA4b#7PEf z{UzrM3i^K*R>kG}D<*Due#hxS*83-a+qkdAe!^bL|29NO&?RyA{KxF>$?lmH>_Bu# zN(4+`_n@!}Fv-efFo`%K^ky`540*Be2vtrI0+1uF0FF4Hn>vJ0i6v^o*a^dB{zQ4lbQiMX_7m0#-}K_!halvCd3=EiE|jU zCQa#IG``bo2F9di74lu^U#PH*15B4s{Oo;AD|mzjI$ zR8dBDjo0JK&Pa14#98r;7Sv{IlhMsz5weMjIXODDInmLPU43Z5&PDzThxE=!%Cczu zfyuF9bk+Po zy*P>IDTjyDC`t}i6i10OXe9%MA;D0?>W+~qi3G7VohDwd=THF0!sThGVqi4KL~-cl zk>5kDM{P}BZm_YYt+uVAEO$)a7@vn&uNgsGO#qpv(asOWerGTz2@I|U!G#G4)fE+$ z16oa}u_X1%Tsu9kIX!=2F^S{N|MhU`M&?2-URp7A@{B{1W^GGvm_Kd8+~U+_g*QHV zMmSkNKexDKUM_DheK);qcYb1Wl#27|`lP}QO$*kSSJh6wpO5nwSyC**4#mq_+t*N> zo0VEPA3-d|M@9ZW7gxlm1nux*K+n+}gR|>4d?r(aNA}yciRn`5izN)lfAh%dZ%R*g#tnY*~m~9kaH|#sT z(fgzpx)~lf&*kN~i#$ar226#fnn=k8oQrpv2J5NFAjK3pN(MnMc$SSgSf`3mP@ReL z^+tz~zEKnvuh)Q@I-pYVd(2sO)AM{2^M{kiD?&FVeU$kFw0o3$;_8GvW%5cQS&j*2 zv{o47_la5qE)4C)RkL#FhoB?+kZh%)AF?-N{GMK7HYH&}Q<(0;^adtPC}1$KvTpp- ztT!R2j9RP0o>YDar`W4os~gQSAX{}26QgD|OJ6;}CD;iuk}tUvLeeo2422lUP=E|~ zlF#vKFW4B_y`fSCGPi=Bl43&Js*2*ek{T6x)59hfP**8e7_bQMhG zNAa{ID*+6RkizQ?J`dQKR}hfz93qRfp`Y_c9j2T>_5!!a>>qxK;KkmKppsC~6@GoK z4ug?M+cy|O+P;d|OTmV4r;BkG;+#-;mxP0Vl>HbCM_bhJz!zADyey3o>lpPq=o7>3 z2RvNe4zvnIS!JO}0?MV>4N=^e-whF6U4-$V(<`Tn0pQpN`n+$$S2+L*SB+JD*d;{qe@t zgB7a&e@?&fwma@lp`XBDDN;FeUK)|H9u+*Zrlwosi)Re>-8ym-=?Xqc8mJgk)dp+X)4$qOOib ze+S>)Fc(h3jaiwm5KV+DMsmF zzfPEcik@ixDcSn}Rn$vfy@|}L2iFe2TfUCB8WZ@qao_meZd*&nlFRxQQ_B?=zbVvx9Xf>-zN<2>NnEOE73m(B?AaJ@mBX9N6=j0a zgz6RKMx#|2QyT~;vW7mxTZ}f|V4=r!A?@PgOO3b+fT$?{Q^QqUMw7u{#Het@?l5{) zr-QO#HfyMcn1%fG-7uS$c3O~qBray&Y5J$M1Zn^rj%pq*`JEb;F1ivUG;Q`z6KXe`LR;sr(P@w*bqAJUmqSZ8SvzBF3$a3IqV5;7I}wT7oFmeQOUYG ztBhc<*%D?1gB2;=*F$aNG8*LLpGT>Z&sWU(3vZD>pzhDbeB7v&ED1jDPc#B_EGufR z;1AY(O%dnmifd1=34l+Omx51}Mezxe+F3)N8O$e0+Zt&p62YTbiUjRQOOZSs#Zn|S zM_P(8!jTV1B5`fdVMEs=)VPG(BP_a<+aq+l zgxe#`yp-D`6uyMpBW%Bv+at1wUe6FIfd^yDh{#7I)Dew*k%D*ddLv^W5!wVx#QlP?AcQKouku2DTnA|?Re;*1<*ljxOU-tiei?Q4fn1xl)`AKLpoS7Z!*+c z1-&V(f@a2igtwZkyxtntKrgybaD+9O{z;t)IuNc!nvw)H4zpJ`qH%>uZ!n>bf=b&} zeGU4j(WvW&PP(+yY8EIEBOo}?qx`_KE0si6qRa6S3-X7)+g zS*;P9BQ5h%ZjO}8OSU;u|5}vZKrA;MCGQM&?5r`aVYp*{!9}3*4Zunv!eDaS*rtXt zRft9w=I1INGP~UqM>&=h3-iFs;qh0Y`Dd({8PvRT7B_>Mr%}34!&#<@@|=zD+%a~G z#VpK)D@*=*OefkI|H&)r=gRNOcM7SwsqKqWC(K*a9|*hCc=M#4?cVBD^Po&IC9^PoF0eu#NKF1VU zM3q39?ic$R!D_OED3s}bu^$UB#dN>e&qOZ8bide-g_mNwU+iZhmtwkK?8m}OG2Jir zGm%R%-7ofI;iZ`F7yFsWrI_v)`!RK?U{>sp!>dFH4&@q~qKJt&n}mKik_Cf)kmIUD zYaW%;5kmtv6ZPCm#@mQv=tXj^(M?Cw7%o%2py|xC9#_e7Q6m zB9yuW8={^_mu5qRzs*rs2L2c0#zq7+VR+*x79t@x{M8c;T$1e&m3;Z_5FK5T?GUAX z+3j3{okURUykPR!=7u_mnG)oT2T0ENJ$Ac0PSx~6VrEF5AS>Psiy62ThY>SU_Uh(| zdIMhrw*H$Kp2ohEuIYiAr+&wGrP+$=ZKbvKvVXwm_(zPZcF`RVej z(EaMUCTK?~H0N_NG> zP&EcJ&(Qcx*o$%QqQ!`29;|}MY}WQbBV&F*LpPjBOFQBALm839!W@Ky9crXaj321Y z7_#W+Tki`j#au{3w}yzDb7Du^*v5w1n(8WwxjFp)L=Sn7d`f-|b>F4R8P>F;5n08^ zbV8S**addDeHywkI3l;|!235`{Y+0ogC)n(k~61t{JM$$BTLHG^iH(3Mrs+(n~)_b zTrO|A{KXA#%3}tibjICpADOtiG;fN|@LXahI<_>PzOk)um2ZfS@zGUs>zYS}$7JKK zNaX^tJN{#-3cAG@u8teeZ4YiK#{D-`;Qs6DaR11m#r;QIkYv$VBn@GDU-YxvvcpLPro5^SjcN=8Su=rcTlo5$S}JY5P% zHdO(VjSYY#=57I!MMOz4!5OFFT5H`~ADkR@^@0;aj-GsSZ6K@lPyu}7a7e5=F zP_+uPi@c(EO}v8f$`!@96C3eA+?iyH?}IB%!TdPRD#TfbP(atBg$w4+nKmT`cZs;0 zC{p-h+(k`Z#$byiB-v1Pr1nem8_m2O%NF!DWbF3W$jQRgFU5Z354P@@RZvx@$1uUa z%~x{+wNB`n44cThZEcz+aDWLyFOt`XM2KeG5^Bic>5g>Zrs)MRESR(=U2^- zPPVLFvUW|+SJCN1J9R21l^d_Q<^Jp-I4#ZHco*<1fETClT)7?-GHLV%ng3bHfKz4y ztKA!yAS;|oM#6cj)L%bpYH-x&^MZ**g=m*rQ&f|cL5-2P0=^*B`-!r^tHa?F%JicA zQ?)}(q9aqHl+*yr^rF_l4yTZM*&B!Uys~EUsqRIm=Ujb$VMC*(!jcaO=$l&IeZxIB zbRT(b_uhj$X0{#cx;AC*3%BoiYNO4X_Kcmlwv4#K9od1bhT_8cOSfJB)b`uo zT(j`Cdv`xI!M*yc8_zzh;!-SFkbN@*7BeGYF_$`sUj!ETJT@$h1CtK_x50wubzKb^ zPubv@j#sQ8MvVJLKzd&I+A(uG4;^xIEAuKb(h;M6ICRC;S=2&+@RdUcin?&zw>bE| z`K6PO&Y6DYxbnpf`(_(f(fvV0R~@3~_3+#^$G%${6bxS*j+6e_nYp0s9S zvM<=|n>w$%Fr@@poe9agd9LP(J$q*@zOuRcir$Iq3loFgt$h^!pqh6haNdJ1e&|J* zg>2fi)D)!C8hKRH349M20<=-35g|1&t0*hT^Lr#yGMqyKqf0_xoT9u)3}^z?pxWxB z>LgvJr~?U?uzKjwtja3Mxa+OsCx3C-zF!|NSjfkfRrm$@Lg63BpS^JXJH@Z1t-oyM zck=w@9ocJ-wN~$3x@*1@8pPhN#f{Z%eQk)~)mvK|^hOaL^nRUe0 z(OG?ET1tD?s$TDuV9OMb)OF_BN3L3M^<-OOV+p^gbXKX=T{wSg*`$OSTaPcFu-2b^ z{i^0|6Q}G#E@cbHX<9U0=$jK@XDShG;s$L8=ioB9l3*bciFx!cMr$uzO{i;R#G%lj zMC7KpeEA;9g07LIa=RkC>_YX!^`NsZI!1~c7XIKK`9wSQ&~4}E9Q)PY8=pC|Y|Twi zNe`|5*0xFwZ!-wj7*J_dG3W7XpTDEOffsIPd#VAxPZrP$crZhlFLL+@a?8c^+ zcA~_FS30E-Pl)*mwI>Z4Ts{~2owIo&p^aABDv14~O`)<)Kw6r-^|PPI#fvgC7x522 z$S21aRD_tpdNoV)O|XF@P!Gc^E1)~DtjHVY#> z-y-}B&C6iVU|vRW=rgjB-G%y$;U4LoA$@9E zNBw>V{T_Fw-aBfkvr&t;BxsFim5k`!hDH;iKJ=MTY%y9x(?{)6g>KNP(hi)pr6FLW z`UxLFC{vpx_9U!Gb{7odMw{K`cPD#89p|M%$RQ=@sEH!OjszO|q-mLapZwwT{jaPc zgKdov|0X~4SRIzkmZmv7n>& zkT#Nm7BUE9(3Hf`qoL1xX~u$s`*93dB5m+GlbWKnHM{* zaO;W{&JhmF?(+XX>;IegiZ)nbzL&+#3E`~LDY0)@4w$9(V2t?$qf3)5LE(i;UqzK^ z$KZuN*8CV%3|BnYMIHsv;76J+$ow+yNYG@9#|%R<-KZ@sr5@*pYKJ7$ zx*;`wB?@CvBLrE4ZdRz{nFCVObQ|%8W=dMHV4#3!aE-LVQu>p65x6I}= zY18t{74d|QDz*wt1Jmk~R54r*DUd)Kk>m+LS|wB}tO<=1KU(ETf99F|$yH^x+0~N@ zTOYWCKTnKl%KhL~uibm*kNfuj?aniNhkPDvDZ?z-R`9RpC#PK3FuiosYH+7N6A>?A z%;|MvRI+{rKr;ZJEJTbB1WlNR3gZcLI|x1$NrifgN@rmcRgN%O2j&!N3dYj%$Saqr zJfzt>%u~dpX%+Z_&;ghVu4r_2m`9UX^jCPRy&=l4BDrrUF+sFkC6+z8;KA#;hC%>`EUh8yUN%DkHRyg*Qvu1U^e+ zxG;tgQb*9DA{1hm8hX{Ss)S-ZiaKd}6mHLpJsZrs$YW3{P-LpC%<)s;X*5;%%WWxs zmxHyNRI1HgXzdNfl;(RmOKplV$!4k$^hBGYmnwY##r{O+<+Ae?`A^q=jW^wr(q5Ga z5%<={RKae&*_rriQA6@ePkrZgG{<}W)AQyGK|eS6^{?!|cN%Jl-#4d=`nmG+=i`&I zKHV?mXf#J|xosN?*$8gZT`Jtp2jfzZQA)LXIhle+M`DiPhVpWyMh`Q^H8eV1Ex7yt z*n1D?xTwR~4N40;2q)pZwIAIJzU)r#uWEnyF4t>FXl8R;~RG`%>ZIMu6^ z(P1PJI`%4^;U@wySVzaqas|*n^ezp7Q+R3u80Gt{P9biac+)1gi-7FSXjGsX2Ek-c z)DwJtuTe~hP{F%er$G|rsOV@$8L(vf7ntZO>H4tGUsOLwy%Bv*^+u+BBL%q~`vQ?p5ULf_>Wf7pG2J5}brO)7Mgh^x zF_gaQ*Bj9o0qPcEX*H>1zUda=!F{2HI&L9qR@1yx)#xouD@;sqMB5{A6)Aj@3C)we zq4^m>d=9&X)KvHf!tTK>9C_o@t=IkSbYe@X%Y8(MQ!ZfdKeuEw7S+1%y}jae&lAUQ zcxfGw_{xYRXlvmYFsKu*k)(Sc9Xd9j@qlxG)IS3llyVK+ivbT+{YC-?)u_SBh7g#O zO7--B1+}^a<^^FvwG2=|4d|$fS-2+VFCmmBm?)4dy1;5&XY~Tw$QmLw8Uf4v^KF4M z3o}`?4LTzYeTxdzRhIjr?Y=BKEIj6eBDYhzYPL>+=Jehtp1GweR7@8a)*V{%JNdJT z0N%cO$$Gcz3xa-w+w;qxuexFU%olIXSd+lu-JYBC=Fg_U`|OQ3Z{2$H+Q@j`xO};h zkB=Pt&bLbWD{lIFT$Hd0H-PhlmqdUEJEia7ZxUDjO&z2Wrg0JCML=vTCe{K%v(S%S zWLKF*^tcwKDADXeQ$6(YaDMuU@DPuVM{+O?PQ{msbYxSao@_#j|{`DH_1$4iU7YC!)?C*vX4bANV@I@!LC(JPoS~ zwlq+y6Exu9lHg2Ts=y-_@>@{PuQLd6lJw)KY$`~=gAfl%bzvmzH%!a(IGu0{WqOmH zNlsWO79;22qp3p#$t(yhOb`vJJU%<*6?}deSl|tvTXa7!e|&%EOMLgwgk(~A*lFc8 zx83#8l~?@vcK!?6nQ1`MfAIy^2elpDEtkmSKl_P1zEe9T;g}S5zC6(R9Bv7Bq|>bj#Xh4ITMWhxcnf0M1K(tVilP20U0k{ z0B>f3rXUIH9@)I|{_T+l6|nd0*s+hFGYwL$!`dmZez^bXBRjvnk^i~|^1lnmqMq1) z^eJIxOoLPi^8qA4LvKN6YLz0*8bum4iY!CG5G@fAC=8Mp()x53@W2pdvUoCDo%W-y zMPTN03Ec??GDmZY>Gwm$hucrv#Ap@fY)f-}9d4%-B>|ZkzN8?{S}#bm6obK(RYtJP z-98(SuC7OW!oc9Yi z?i{%xYJ9bB+jF}Yo!T>xpU=N|>*L?}PYvW@Zwl?ASiGY2*5}1FKf{F6~~@(caQr+f>_>S(90V zHeN+}SU z`GM~o92gxQ*fhAQe^p;kx-T{>Cq2E8rA;`{v;@KqU5G0XHZy2akNT8^BLDa${xF}r zghed3LScTWR;R8fT89$~Fxdo$5^G~wn?B~!21#Q3i>W<7z0B!z)ZDPb*Rs8E|2?Z- zzN>9*TGp!OqJiP%={0e#g0z&mxkkHb?1y}LR%4PWv#Wf|fwC3r@>}`4HdOd?Vzz9Z zzi-D-;o6Akxb%(fW&Zi|{blWE`zz*@q?gorVoQbZ3!9q@@o#lUZEj#`bA}&U62X9^)bRhN`M3Kt)F%Bk)VjFrvr? z32ipw#5?uZ5dKP-0E;k)iOn|gCMU$Z98or?=W%?TI@^TIO;sIn8oWw$puz>XdzB6i zg7?y29KQ6I*B(Hs>iB5wjaT*F+p_HY@-hB}f#G$>VmH6@;GG{G+VbpPi`O>#y@QJi z_WS1x*W|A|b{migJ-TF{uz@?t#c(NHU7*_SVu!lm>+)Q3VqC1lZi_JLxfnjiptN72 zcLg%JoD<#@-8mU5@MB%adLFsv(*3;Ll{K)eYEypY&_Qt~UIj$POwZT-Lfj-qg$cPs@QhyD-1F(MepI}aR3x~}UvGXor( zt{yq@wPTz0PGkMk!B){N?RBl5pQ&XKk^-JB#O6R{C$Vwjmmqv z!Z0&*-;1v6edNgA{c>04z_QBCc@+o$&aQg}Kza4>^>|G~G+uK-LBf2z=ApzIye8%! zJc0QK-`Bsu-g7>ifAH_UD?}6WGr#Zn+Z~4%-*fqnFUT8w%NP5Hb4vI8j(1eo z)h*NQt-EaCn!68=80?0ct!;U6tx4JZSVCpzGPOU6c#rgJ%mcX&G-WW|p=)M-f@tCw z^f@xP=<~0rpEEwf=A_f-nTY#fJ^TbGbj=g}1n^PN+E4T}6M@$1P4`vcf9mJ=O!EwU z_mpQ;(0hbEFIqkkoM zmInvav&f@>6K1NiGiI8*2p)hJgNt@n>2&bB9}b6O*ypcJ`26AEef(H=dFWk(-$U~y zlzU=lla6c8r}KxcFVQ|XCx@>$EDXLc`&_-h?U*w`exB{xoC$*cG;8Uerd=ytue~?s z%7nk)DeXNl^Wv+a_tQscpA*cL&t)^oDdls#7vMN0uGlJX3#|dHKMU@~U1)m|Z&dDr z^_`NsRoPd@wFO#BigRK-m`urImI|dT2sFi94eJ{aT|@gXJu-lx&~&D!@myhkT3LFT zBH$veihv_mAcl8o0#1<{Y{X2sd}I9$ku4RHe4502_BjYVZ|7BOd+%v&drOmeVoAwR zL$Ve5g%#TyY7fM2`R#*uet6}7eaq~%b>4be$KL+^JBEd$`Rl%T`s_zF8(ZDJg52SG z!#nn0wt~@>Tsq2bO!0XT*D_5E*8m^(dkFAWc@OPleYz{Td4ZOKeE6eb7{R*_ zpM?%JQfMg;GUwKz3Q8P6BpY@Rb$le>#wMZDdy}B1s0|Ei8qMa5&xGO0N#tpfKciL7mtp}28@w6CTvuR0W`+TsUNI_6ih#=p zIRzRprhx*@H-J6Ex?)B}78Ov#;G~2YcVu={wmX%I1An%jvL1aM^q| zxp!6astD%rLcv$cs%D@zF#<5+`TM`Px;S=D!bVF+Z^ox*rbim0ENgtkw(Z?L_L$ea z3Ey}lNMzyY0v;eg$6#Vbl~g_Ve*2kh@-d?OtImX=8`Aqa;A7N&#LS!ybhe-+h!;vG zNu81vQ8HkuJ zI;%YboARyf(e`#V7X z{>L%ZjY)?6ro_7P4`zKKy+x5}bFUa1yJBuyWW($(CnQHe%X;B)hQmxA0*B)fNDTyP zfy00zB?FO#gD9M!vabovt*O~}N^(M4Vwwy12de^pm4yma1jSqH-_g)rBhz6v~i z+V>CF#hU{Skv8T1LjeSK4!k{uN`i+#UR_))&m|_rro^Q<>=9;zM3#+F(a8N}WjF$} zfWwXuJVEZh{XGA5X4K?WT;9Csx}Hb&HLjB6w*#{8_?W?As#BG5rk>ZXSQuZsn87++ zCx_Iq0StTxWns00nQFnGi+W#0GspY-&-1>8c;7RbYnD}w!3Jx>>z>)yv`QzxT`vm9 z$MgaNO7r6565bT-GLbyY-@T~Yh)h_2tG{-a& zp6x==eScu3k2T=a!@z_jK+X}|*e0_e8U>MtSs7>y1-eF}`c^k!fxinSDIsZ4T!c)= zD0rYMiYu}+Q&SS+oQ1B!C`XjT?M}5?Vp(<_Qr{=H>e`AGxPWvoP>XS^tpx0e$>-7S{*aKlIM57^o0X&?mzafd@uiA^ff&T zzZl!uyMOhvuB*P>Y4e)*erMB_AKv-p-|Z2O9{eh~K5#FTf4dugg*eVjz7OqPl?(#r z$zkt|mxOYozAw)$;<=Qh1oTyouz<^vpGNV0D6b2z81gGZE*rgGR9|;a^ZwGxtE*bt z6XiC3!Q~x`uUWM6hIjcn5xrHV%SEBk-(FR^_|kf#Q{T3|Vd1egYp!YIuU@crHN7j( z?7+o-T&t6m&anKbvKys2nIX^ce)eF*b zIU=oG316bJXL?yU=hSy*PV|goKZ7x0k}EaCfN~!NshDE^;ahLDZmk{YAG~w*U9Yud zRrrOnhK_B!H|eTG|C*YuTZf*UxA6JcTTagH_Zw1=2Riq*j13mG7^*0t?$nXFACqRZ`3=NWwopbR43-gFB6i=%Ny(cf{i+3n97OJdJTCRA?lz3awaki!kM5w3T2K+ za>~mgBO$8M-sny>vxI3@?xQHh;C)QlyQnMGy%fd=$Fpa1zl;@X>M~d%Z<*zV^IS1_ zM6HI)33S)Eb4Go@Wo;s=CjE3et#XklFo??;=_yIE9!GSP6+{@qWt9lwZ_G%`(5Lx> zZERgdA$K-$5wzOIfB92OpIvP+R2FF_Vb_R~88vOlpPu`-?sSmiE0-nNfIUZ-r zndWSY2}H3qO~Pv&2IpfOhAue9jZ6(F99e#pkURmz5UoX=rQf zE)_Dw)`jiGIjMYVdR+ch>sKDBt*;png_7)fIW-NH^=UlsuWu-?^71`#d5P5xD_S#` zISqlpV>pz+70dbTzHG?T050bOkE?;tj%eVr2pgB-vOzDQyY4|~=}6T=WJ0gkhE0j$ zG`JiUiD(~k<5S@h@VFuyq8cXPvLeS(ni+oPSzW1M)=)XV{*u|;uG0ML<|JnK>F4m= zEN`3fWZiAR<5q4(ptlZ9(p&3mi;bY^NQAWGFlkl?4gug_VAbGb zlB8-pkoSyRL&`TdR#*8;^K%gcO-lk;-~#aE@4lGSRs_~n9qBMc8G+?`{~BPq3Y@tup5BqoD^@2YD1^{Ee>mkU#q;}XNmeC;zLNl# zA`V{WBFyL1BTz44{4P4~Ls$W41f+TZQD7DmqA-)66@zGiNJl=kFPDuoacH5XP-sex zGI(4{r33{RP`Ii!tR8Lqa?9hdY*{(5W77kxH-2+dOICK3*|@T}bWPpSO@W5dOPZIS zoGV^iQgQ87D^6HaclUH&)?U1-p{Ok(@_IvYVQ&pY)at&4s{_?^mRQhp1n4=RG`D;@ z3+zww>1fA*7?L>5Ki7_f{CF1Fp`(bHTHmzM#5rjkWG+E>NzGIqit|t z*Ol{X>f2kl)UH3B9~b8X)uj~otZ2>7Y3|6aT$3*q*DqOEvCQC{=P&5UZ>UbpvzvN! zN%iH8b^glo!m7+PMfViHfxB?y1ciaT2nrHo;0{EONNvRw)MK6~b_>uGd|qcpo|+!2 zpxy-ifC?u_{bH&C{;w(q<>EI2yYCr(>#de8^#lDQ4-8%N>b3*xZfXzg*u5zLt>Bv8 z_M6rTy3rSIUJg02#;;ctgDc;??)t};tzNNtq}S6jclFrNOHvLb1-KhQK+5=( z3GbYdZqLt5Ra08iQYX~Bfzz{B4kHr0J2XqY@>_2W-?KZg?bU0B?;l>hwth?N_aFxX zpofO}r`N4JJ-_!F-g)wtI zhT6f)yUVg$I@@xJ2OAp)i>rFYZ#}l^OWl_}$oRLup+1o6uWKx?DjQq2Y-de_rbo5{ zAF7dcYeUL_Kw~wkkj1Gb9nx%9%C3X1@()VxP9zz%7Owh36_OiVJhH(!GiCj_61$D5{#THY@%sQs9CY3Z9sefeJ>Y zh*448!14$oH4PQYECz(3sDpp^?NfW!45qIAk2PKQj1KCpdfsJ-T2i-o*RBoLgxvh% zk+}!<)D1ZF4nB5p==#{(k8Rtpk6W}lzhg^nLcu#p)kzk+J}P6;VE+nXgFbFy=g?-K zQFNMie2@^s^j5NUg0Rd5|My^CpGbvZYE-xkyn#HwfFYQk5ljyuH>7SIL}Ca@OC}9K zkwl0SU^(k;K)pU~v!KGVqP#FaC)=AGAM1>^T3}L!V43MJMeJS)-!mO4RbT8cB3dBs z1Mdv%aQsONbJ8emm%n!54MT&mLbTp7SiASKxg*8-xe3+{yLK(EU1B$&V9%-_9KCnp zz};(82iNSq@$J}{gb#L@oTAaUd8l(?9A-PO=pS5^5v8|V0M~a45^A?}7r{BNr z*zJUu6o<}3{ImjiN!g!pk>Xr{BNYx*hQm=X!mBqE7$?CaK2R0D_54>bX)nXe(vo~+ z5hf?by2z>kEGzg5vu5uBmJE~^hNPyG>G_DV{)Rx5&T6gCx~rfm1?jlP=FU!kmfxa7 z)`w9SXx>!Xe{IF8g#*WAUu4v^EjCGVUshNUCD!v5Rn zHzP(4_(4wPf&csu{DSZkh1tSSjWM<|Ki2D@ZTPXockzFL#dQ=KU&JQ|qC-?}5gK?i zNBMG@z|f`{(aa)nf(6}}ClS0Ey+X~Vb&Pf?w~?5r;=*n||D(96%wxJ|1&}B-1MgvP zG|gk+mgA6NgkiC0blQi8eI|_8#7k1Lb6RZ;&$Z8QX{uRNyJ$KFV^f&*=VmZa^fQVv znNDFvnWc((c9C4?k4a1Fi)%fV^&M#giH${@FNzDvHzX!3_LY=2tc`QbT{EYx1NYB! z%jD<9Hz6;}xoZQqR9=*9uy)Wvgx7d{E=fgbw4~5NauP2`ET1YQ4JM@Y8P{qGF3Z@* zi29jUqqPMJppXu%hm~M382j0FWbfPYBIR7U+mjZP?#qZao04+j*-L|iSJA{~!@(gb zJs!I_+JDuBSa+0TRDS-j^=Oqw){qJ-Gmq$}u@9aO4Q2)o&R>ljs4GDDUbCRb)Dkn#o40CDz*ba|De#`K z*EwnL#e6UI-1@xVOZoHRduRW>kX0ITTo6o}4n@E+AOcAL1QL0!wx$}5k*E_^dTMf# z8{GjAYH#A3B7(6%ro7JvWt1E~;8FbMV&bYb1-XTL<{ zh{wG_d~aVe@3zT}yx-<=jgGovtn#aZ(dyxSo5ORRU2AcK#1EQZ(XD1kImVHt?2q1I8GVx(|6 zJrG%MM7cVgNlNzv6P4x=G7m6~ff^`cg308)Ml*jIu>S&^Dfb)$NN6+AIrPo5+5NdT zvf4mkbB52GlIF%O*j@H03!1kvY*q`BL$^>)w}7y*b_wDgXKz}WoV-#jz9e3>%6EMf zg%R~bLxN;VkdDd)Ue@?2cqSNN^7n11Ch0*@*$hyCW>D&oW|11b;%!>tfF_jwAmAYdFfC!*j6qgzgG4g^PYfd)^~x3L&Oo zllEMrMcIS3_iCa@K{7<%q?tSVS*-Ua=TSK(M@)1EJ z|N4qXPx7|V8kUcU`zF{E9%y?Z%^4oB&mq^sK0ikIp?$950^r9!2mG}Ao`5&qFYf?Ds3;V21sgE){-^ zu$PCi}MMHu8cBnQ8?dnIEfToT) z)Tsz!dkwJg3NN2X)VLsUvUU z11t4XEh)T)c3ADDrFLZ~u3ubFqWbI`B&zxTk0K)#Q5_likvy!65E944w@Fx!M^bks z2y4E(w8q9Sv(=UI`$$*|=YD(c9XLr3aJgJLcbMf;ib8f4dQ*{KkW}#uPB2pb5M@4~ zd72TOA&@)3!w0*8S=~YJbnf{U1``Re9Ek*q->*17&zF{x6c@v?*>qekpR4$z(*--L z3RPn_aF3RjmZP*bg|j>L#a=3?MfWb`N$6OC4!o*1LHzX@`Bx9S+iiyqjnsH*<1e$! zsoNC)2yeQbPvD)pv#po@e>b#Wb*y)T{HO8vg*O-7x~=nTi<)#T|hVS+;8QT3|taQN|YY}QLlkpe1{5cN78OjIKr zfJQ0A9a!TjbtDGWuCL*g8d_x_V}(MnXnlg&8lsLUe74&aDWK8&#qXcUXeRB4vq3K! z!kC;=ps&3=q`>4a#?X5BNnw+#C(xdtota92Ct0{L(aa`be1;eeY*!boGV-;5*m1J`5YOXX&pOGw@)t?o7QiIJZ@9hrbu zEBvxo&51&3oaH+H3uJ=~;POYxs}i(F>MKNe4=##o z(*=H^o&JfU_?-i<;#dDYB`!Yuu<(uXLE(<^PO0nDGe`OV#2-8iI>f1SN#Y;$)p&j+ zhkkoD9u=LR>suU@&`-461>Oi;6`{GHNF!*sn~Zug z*b!kR^BtVQ#2Jy5iQYtd93OLCM1BZ2#_;u89SJkVZV&B~hz(zOBkXpz2MP+lY<8Oz zyWOj(2#Vc45jR*=OA$js+#opw)L0>0t(fiOopWRONE6@neN&{Zx!Gnje_wvrEx5Y$ zZDrJoaC{rJBJ{t$S8wMRN9mXH(cK0cUlOwphr+#eoqI=j7P2Ug^Kt7RS5u0W4hG2p z@r2Ci(FMVzVz_d_Koi3!-U0{(9Zczr3{TwEXw4cOUF2lE(eZ z?)LKCd~Q_WN4);hfxQp)E$w+k{^;9p$WJ}BZPm$p+UhTPX3OL=j)l%hlJ-0&VBz1i zX-)|5-GuDwna)YB$^ZY(3H_ekC9{3T$)<$u^Al#X2%7DK3#B!sxbHgd_JBDS`8*kj zlamxxmE;*>^!pn(r^I7vlQlL#u{@=~%P^};s!Px3!Do*BOPn(aS0_^bDF=fwDm zB=hPen@^uQ?n%)F3g(?$+jZO4@{Www$V_W}<CH=4n{~xLPj&8lzdHDX z?bqBp^7W1KKfIkgDze%H%hRzb0N46$dHOxVkpov9t( z*!bcHDn8l(!y4|!fXxPWoXQ)d3gDy`hbrY*4yEoO$c}^-3=HIN7{F)4GG`5x9gNJp zL3H3?U5_efzsqQ3`L?uIZ-j0ak><`!_h&dV945G<9TSp-Jj@@hBx$I58A_H0Ycow6 zGEfj+RrFCwOG)nzCiLZowXp zDMm=zcj>o=l%LH={RMcqLemNiuMc2o8i1qb7!ZdmoiScI9ZunSjPFgzOqgYQ0kK+wpF7R% zYs$=Uq!RpK1|S0{lW93heg-fQuFvoU4JHF%XhHqaIs#B0tIj5WEdOEX@%{Ufgcy@& zZ^!WM%j?%}E~xUSMIHWj?^VqUMm&Z%Xh2u~XlVDV`)+-7_{XQQVF8U!jyoti(CX2O^J?s6rfLtj?a#K&=_6t*mb+Fzw1Io6Op~NOGkFTWM(v)p;f}wH``riGvnQ~S8t~I35|j3Id)$L16n$T z6Zu&(8&!qS;*RhHk%dN5QEaLk96K-5DOpswq+e@f_oJWR@dUqV$(F{%8QIRF)7!UP zTC(?MJ~GtEPX6FK&wmFnMRKl~D7_D_^ZJ0z1bYTB64Q`9ufa4Pl1(^-lnlU5Eo~*) zg-$`tD#_$MM^9Y2e%$Mb5%g&BiPo87ln|l+3Mw#BnS>+>;00Nq$jn3`robP<`=fEU zz>PxMOS&G|W)gbDA>#)P<*+HuVNc9(8LaW>OQ`qc11(T{*QPpe2H+rlyG{STMs6_mI|jA!8sq&&i)LmKwPDx*hY!%*zpxpUuBXeD?* zzc}~3v#%-lbM`g59~f_Lm7ct?Z?n0tJ%(yt#?NU;Nd~8bYY>Qr4hH<-0(f||@CAbZ zoQUp>nunW!ok2Yw4+Xc(xO!F#nG-sYn30j1%5fRl8QEExslHTSdK%~DyeY}qc1u`? zxT!>`5F{0Y`tUCi<`l97QI7vn*fxG#*v5!c?3ji$&-xi*3Resdzc)NgF(Tlz?nUW+ z%u#qwqk$^em1P8--~D|uRJgGGtmMveDHhtiTe1p0cU3oL}vl7qx34U zfI{A;4Sp+&F?AQ}7^va)SpXOXl>FYr^ zyTzDt7C?Os!c7@Kp=hr?HQF1Up6UR_2B&ktFm@1a`t4pd#G(G3c_EdY8zR~|K{cTj zy7o@_0M?DuUatN&@@V6$ORmR zs9E9si;5Ho5!p40{V-86s>9AO5B!)gk5f}t2nTX$wE)y3A$_4OvtP|Zih-oA&i46J z@7f;Yv`0m2$tENeCopAlGHdrftGo8l4P&KYjcnf(mDn2@FyYQ*c{VrPxH#(N#mTl! zxVHqW?nSik)_C|uU2Sek`=_^A6TpSkJC&YbcZx{@3Qr0BtlTO6%r$WL2AqCL;LMr0 z8nh6H(v$_&Nxjk32;OWSTws!*Z*cvTjk$J#5tV+Ff&|a8s01_V7HK7CK6Kud7hI&U zcxRw)P8FK>`|Nfn5vD!Xs!(OP+`Q;3lAcfqX?B;=*CTBRQVx`{i@J(-a8ZIW=5}5W z7erbBFl z`u($i!o_V?bt&}#6bkb4TxUnyyt%Cv{(^i+d@$>jJI@xblEY&AC7W^kk8Q7%0euO3I*V`lV&M zU)pe;Gw!6^!x?u{!orL@DdBU*o$=?}nK1Lt8NQApg!GS+U{lO+lZ!A55q%&BLQ=^r zjDjXW5Evu@H5K4fS~dW!P*AOcs8ZQzi7-c4kVqb3!2`Hi?k6aIt}iAC|2tnym+-m1 zm@eqw`C_`m&-KN0zn}k$KgV|}$f0nKd96*24S~|)oNUtZQoYnnI?@VnQYYVOQ!2M8 zbtA|)rtpoRkO?!*sD6tX&8|sIL#&!pSZdsqTvtE0tKDg;kbe>|Ipv=uER>&aOjx?q z8JoGUp>j^SQK*?CB$V=9sTGw63YHBnKEQucnvjt@(zB5Nesyh{ywQN*V__izkp})L zc`l|v6cy=&SpM6&?%IWUt&Mq!zIe^{?ljdV77q2v^H=PkMvh`o<}Bn^aT~cW1h%go z>|ffws3BmCFqMHRw3UKm_&jb%pQL!O45`Cr6^Ola(^CREYfVE*#E}+_tn2Gp&<@tz zQe8zBuRF%)Nux>I66=+(G!Y5H_9B?FsznDmGEW`~r8O~tFHQJKfBpXNZk#Q%F3W{l z9puyFyPl3pu{HVk)b;e0@s_Y3ykNT*;Na6(*Ckoi7N)r=%v)$aWK3|)&C6+HzX<(Z z(G7w&_5j(}Jo!sWT1PXxO|hkQ6BI1`hT{)S{0%Li@S&nWS@9KQ6DMWcrCHd-Gugrz zTpH_`$p$6>D0-!sXy`;7APGo8{+?9+3Dw{81oI~d6Z{ERP4Fhfqjfd&CX9w1n`tsE zNsg3{c#hYR3qf6{oL^fq^KTVExX4uy<|F_>jI#g;ISFP03LFG$E(jQ1>BtD~0C{VF z2;Q@gTNf~a4JGLQ@7^{K2jh%~`><44Sb4$Y@qg>)Z|r7xT#EB87k`0Cdrt+-4j?1y zKsbtxQ~oMN25@295mtfupy$^I=ayBo`}JZCI{wod;owY*C?J=HC=eW;$VSoD0`K*F zXv!Hq4>>wBu7bm(NBeR?#6TyEF0p68zDSEQm=be&6njv$N4TO0O%laU;_~i9KE@URlULX793{buNEhqwLF(6mkCwem7JS@g=*>v9wPFf zFt-CRpIsxh0Vl4ddcr&2ZDY@SN`hF3W{Qi;w3D$*}9CxD99&j71gJW^91FdNP8^t#em zMwi6U11|lbK2S!uAyid3_(=V)JeBgh^=@~)U@cm2Fv=xpr2vVIAq+1@>xu{`P%Bj> zt@$6Em>z%s-^2-pWy}I6bYGhWDLCLZX5jn}@0^x*a;vBEPQns*7$+cJxfWn#rPLQ@ z6(M>?@b3Q=N|&nIS}?JY89J0RUa-VPQhtNm85%abI{0hskfHm{*cN(F8cZM<2yUSInCv z!cBTAf&d+${{!i|Dcm31&k%B>*@jMthn7ZNtLXdz~L3w0&AaNhXKk%8O zUA2KXEP*Dg1f`V{bb1h`o6<`D;Q5ylxnzU!p;bJvhdN`@4uA{;QKxx!Aa91XcmwbN zMC#~O42ap{wHWn8;hZU4iZjh_&xkHHk!H+NA5-D~C|A6z(0jB3bJCXW{Gqag+bkZ7 zrW%K-%g^t;wk}LNmVX*ln$ao(@I-|jgQu7KV*taW-6;;MO<(>0TD?6sHsMhyJk;NYP^{T5U2ePBm~Z4 zJ>_0#uZG+UvN56%mrg{9IsHjvrTUts5zG(?#{Myrc%sGe-g_F0y6L8n)ggalNInxX zJov65zFM_C{s*3_eopZ`n(_Q^!BgOpgY=$`Y%PtxO~mwxyh~9LfwqO9LSA5C_B$DhEVE95i_e~1o_M~U=#&s-q|Q6v-klB14%|xo4Tm zJ;9Ges}&b8l`UfAt0e-dv{oxj({NsjQdh;xarf}?*3oBJH?mtP4vCVfWH$7SRC&}s60|bx*>S6vp6J_d} zQ0xd(0b+tG=nQ(pT4ZOkG{rTna*lLfO=8G$jR$NdN5V! z=}EB**fKDgn)M7yjiQ`Hq6DS64HaH!-mS1Gxzb5U_~t@~6fM91DrBFvvC+y$8eg^9 z`sA;)+VIaJ4Ddd@%`HD4X;>!zWvL-bev6gOg?GMTdNb9|52MeGmwQraa8rmZT)6Oq z?GOkwcx$+GRr_%*26A(plHf3a{AQ)p3X{Cmeg(}qHQXg)dW zl<#nk$PWz1%4ew7U9-V;9@bj}Ri-S4@~Bcvq4g|*JVWu6t_1zBV!1@T+)o2GRK9ag z9$^tRA(nj0MZh&92iw3H!J`wB38-M#Ha~v9G-eQ(*4;!T` zKYDaj>WEC_jlGCrX>hXwa^HU}8HHvVJ}!$QTc6#@XYz3$Z%1(d`JVMu;M|o5p{C z#O;#5Z*>cVj)Kd^SIHf=Xi*o-w}cntGdkw~JI->PfBl3xUU0cld%>fN1*eN({O{WV zn-XUfIrLayaW(2I8Lts>C|JZIYJ%d6q#0rqj#LEg^lM0eh0i7|>=qtN3O()o0jHi& zD36p;6Xp=P5k?9+aVI+^simSWVnZ6A5+V1x++*Ya*h2u@BmPu;|0A2@vl}B4L}M?& zk{F?z$No<6@BtoG(&bz{cSP~Urjy@P!W>DMp`36TXn~bLbS}`sp5g))b5#yGaV={5 zQHmRi&eBRi;{1!cs3#1fsBXL6T^eODQxp|dR@9RLaa7ET@uFk7;xobls}fw51KqNm zS>7~|KYluYplN%*_-CDW;en3A($9Wc5b!1^djkbJSI<1SF?etqPP`sELo9b)z>J~- z=vk=y1&u`Fl9FJ$A#4iAnn(^-ek~5kajfViOsXX-t2mg0(=UVTl09eo6^hIP4n+;M4vNjMZs6|B%%o4 zW3fUCce#cyAUq(~Ef!4nwuY9S#~&Nk`huaklki+Y^XV{Lal79qgPwyJE_^E@g-_k6 z`{y(`?zDcycx46Z3W@HMxet{?+01aa#=+;B1PLy}>p&k!p%37BAn15i-z%!TIQ?PG zmq0};+yEY|xLyfJs*{#Kc;1x+t*ns4fO%fE4FDqiui;OIMiKNaOuH0U-lJ|nA{D`p z3e1wZE<^AzvNj zBCH6J`CS=Iz1;5uHpTBMiIIRJ*)Rk=Fc-!5s%k1o0z>I0b?89-NeuE4bV(4yc|@}qrR4;V{TWQHwAuw;!IGT!VG^e zTE~Q^cqmT72@II`%dcuE`t;KwHY`UBkN-8qi3R&G|E9u>$uFkiXc~7%5Fd>Sg0`PA z;n*Rsp8>u?lZuqgV>p6Nc4&QUK#vtt(!=l~9zH6~#fWW&%VxO`%{5X|I4&hE1v!r) ztg@gt%>}WF{8KYw*7&c5=gx~=x&R8I2p;(=eiie+a14tGVi>}X3Wi0114W>MPcfFm z(ZIr)E#`Go@lADY0pBieFJa%D$>^j#VhR#Y{q{g$)|(|xhgl0FoW*Rm^dVQCYHcm3 zwFR9!HQ!VaL9Mew(8->upvEu_!ph}gzA2_e%!s}JdS-%iO5}>s0QLdQU&}!ycQe-- zXi7m3wVF7DMKI?EZdGtnnpA=Ib{u&$;P`-t!+^oIREaGNkXE^|q!c}i)}3G@fnZrVZfQc+CHeh?lwWOFUPa=u;4 z*|T%u%G(BaUez*BZ}g@$4&Jc4XmQ*(-)X3+Y7NxaG_+Os_9Rs1Em_gOO0v02eSsAi zLR>EDtf@%8U0buX-QR6?M!7xJDYk+616$)PU#VK&Ik&d2PhOgl<1|Z-hUB~@OBB1D z|3ug?ZAD!ke&)opuj%%jOE5gG!lsi=qyP8e17n0H(u*M&c|ISEez)J|T(GY-O5$^N zjJ5TgT(NaupxK~L^46`oc5B{(*!zB5Q|WK4sjjSPEnl=ap(1Nx_wpXYfUCq?yEI$p zE)!8vSX){#RKL2lWT7QG(&4E{wk~g7xjx2nb9qlkm?<&$`AP<6{(;Ro@^Lruf!GpoTyHl=eCD zPOlDquimD8ZnDvOAZ#Sw(~rLNd0Z7LlNX?azaK0JfO-dgem5Sj~PNO6Fi4u)^RdFj z#XIHu^1ADmjSUX09Nf`&)!~A^rm9^l2M^YN{Mj#c8RBAJZ}p~|+Kh2^1)@GGwPvtw z;E`?jUfOr9GbJH4YoNk&*QIwomFti{YS=#a(5{h_UmC9pw5CRyQZ|{#Veob{SdRkepZ#pfif9emVm^VJ z)15mDc>BWOeMKOuy$!X1gyan9q}vgaXK7s9NM&OKu#GT1YkKTPTIWPYAHHP*S{LL` z0S;lI_D!AB_1T{$09hBEUN_LH0U3P^LP$LmkW*e6eHlG9 zKLCvH1B`Pq#khvMIuMZ#=W1Cg=)^!QgcJ;#=#Fj-!a)fWr7;rG0QbKUmMgaox|^PX zyOE6R+$rlww}Y>m%?wcoIne-kftrxU7Z+iwPkB*IaZQFVH6_+#i@-o?wGF9K{)PhN zVy#zD5ypX#hz(83qUuz1rp)wtco<@KtsR)qe$UGd1k*1S@4c~{SghkLI5 z*X;$VU1y_e?!0+O%p3V&RIbdA5e}qO{^rShe@XZ$kem=-2kf%YNq1);G6PYQoNU3M z_wZtH+*n#TQ1tLyQ|yFb2c8=OUG~&f20+d*0dlkrfU~5hJCND!OgCasiVq-A^bP#|O)}brWdgbg1-5&XI=uJMZgwB?Rvt?%BY_897 zHlK2r;b&8?@&ZSbA5W?uo;SQVGa=fRZ7Fco4CMA#F4;rsUI*JxA#{k4{-b{W+Jw(T>p!0GIrJZ8J@lXO=Rp5aK8ODE+tBxt{-ZwM zM(d%tgZ`w4K!4G2eaxfDtyL(NtmQ>>4Q~;1(1{7-Nzw5adIQ{L5Wb>-k{obUx2haAE4RY9+xcw zQ%p_g#UF5{9Ns!~1<(vIxf67o3A!bn4RRYg7U$0fJfh(Z7BR6N6&O14AUx{ObUDzg z9_j3QL=uso%nI710S;}Q7o}%lR2_yo81+fH6$lP${VJI@mN}&lE$OWlY$h)ZE^6j* zXWfCUvEql{-&U~Z)JWg$bLa1BNbRsU853=Vfgjwv^WM?oXS24}UU%D>(|H?{b>c@| zsRg(H>)M`&PxN$^uitv_N?YVZqM@~5`;wuX@BCX#{x9!+@;8+!BomV`v-Jl$l*z2he<67@y*!2B zv{9}iwKYU6j^%Re^+~U}y>JrPQI)AU3I6t_NlpU!FMMy_(%Q*Rf`QBHkH9@JcB;)7 zUmNBgxMzRJJ#goKau4uPfgN)X?}U3m_`c>IDB0ZExtZ{aY=M(3AE<~c2;|12z?bj9 ztT13wKM)T-7PVzEd@LN7lf@w$r@(I40{{ryyV9s}Y|u@j#OxVZDn!;&mL0o5*eSf3 zefU&O)A}#0Z`$*NJqI6Kz3SE#IsVh9PZta{9=Uw;{_>t-ao)+3xhn=M{N3|A2M+X% zJ->U^9Rp)uTAh$PvS`;6W#iX4n!d1V_@>Ppn^v|o5KWwY9lGp~4G(h(TsBu6D8Q6o zygOo#cze|KB5IEDAARck>jdb8>8V(okcZdT!|On=PyT}`9=-}ub9K@!o&7WaMC`tv zs_yQp`TOSYJh-nrF6GHpIm@~iHZ7=HY*?{miEm*+dU;88S9RT3)Aq$H2V-L96dbLQ zZ;P7ORkf_XwxkB__z2cZf#_cWjwf^Z+}LBWh?`OF1|h3QgYw{l5bTvfVw9ud!e)ey zb?Yboa=OozPD_ZCW@S*cfzFDn@*x0RN=R2As(8wLz$&IX>I8+niT=gad?_P2E z>J@ul-hKHuR}X%9rE*ilbB|m;c3ICACr_SiIJ!8nrg!wj=;r6bZ|PfpVYhZm%Qr4Y zTme0%KhP}$-zuQEX>Ja@AW=vZAevU*?L@=J!`23}&$A3e%BlYdJ($-~YgiW#NO_ZEJ&`tR1V(%o@C8Pu$7v`~ z0qO>(*aQRPk%Z#ukQf9%B5r?yAY8iYtHa;BV{@-0)otdV^kulrhW3YUy8FK_KfwzR zEdAs_@GnEk?%+!tmRF-h<2-oV6crP;FB8(cHjvIub!=W)a&!u`XnIzg8 zfmxYQ1`W_8$uPvyfu|J9Se z##Hw%{>e2?apcZe1^9{4XRmgfOrAHs_xd{)6Yp?{_O33w=#am`r_c}qn&JE&D$26C zs{+wTuHt>E$qDgL(K6U!l*F}63Nx!`qErCwa)Osa$I@|&;{z;q9e(yDjc?t?7qME$B)+z`#bGoq^;7o?YiT8VshQK z1$=UyRcIO&g}&p9kKSju+_cCl>1~(p+IQF>ebpkEi;B#A#LEr`vt!X;T3Si@5wIFq zeuR`eF+T!66lKkl;F^19TtlkY`{6x^e3*S_!GCmiwae3jQ7`x)Q#Xnw5x*b~vQ z`iSf!@+Z&AzmT6v=5LOQatBd z#xhuYwLo;lK8v!?dvy6@Y0em=}O%)M6H|7c>avA_itW1mYtJaFEjCNZlu~$6P>c z4g<@HT~AGvvnQ@LfEO7MNDJ-~1h#gQ9o2K~xZ|Gwwfe^V66baHx}Br) zvrpI(b(TEL()#VR$Q6XGHR_g^+Hol87E*`Q3)}m60P+}Bn;sZHQDOiK*2|J)TQ1lBDDr_-R9|{Y| zB(rjR^M$)_Td}4#?`fYqOnRj?hfA-Ij@zbH!*`LCK1q-A z69Nxt7kRrVh-cO};f}~JvU{MBWn>vpDd<2CY9wUR8Fi#f>e5J=#4If(aHN{ssAX_N zRr6-sD`K2|53Kn%FJ66v{P(Ap{RdPU$?aJdRB7ysopP&CFPJ4=G~bz+_N5Px-1>(D z6Ez$8zwDAOf=ALL&4b9*JqiAt%~f*80(SCQ734b*J+qk60*5?TB;j&kZ;Qw_180U0 z3UM^uXvk}|ev;QJOIsX_x6R<-g1;&^hvzUjr!u#a9DX$OMbU3me^pSwVTN;X&ctx8(qtc#nd}!nSgS|gGJoM<1&dyu^bX(7(g)F_~93Y z-qqFOU{Uw|@&^x``sh0HQ*@pCy>y2Z&v_7eoP*vNm^Lc#`RE`j3jT~_^nfPNQI=U8 zlyi7xFv$-hA6fWJ4*&@=9uu$p60CgcZ~MNLlN0SpmyqD9L{`!vwaP1augBnZ_>x5A z;G@)0eab*OnkEjKI5iUeSaAczc-j!AordbteQX2_=Bh9`tPGvLm8J+lMXv+fn- zMJ=3WAAk6~+o)kHY;Tc6;_a00#9r0IGx^&H^HGN~<2I$Dl}1?xU-4qrZ&D}gp#Y&E zpPZ7h7-kt^#`lSKlf{!O*-P!rU?Y2t;xR$pt^hGp0lkuOmOqj|kY9Bh`NNU1qLJE(;XW}RN_+#w{i)D{7ot}jxmOgv;0ZY* zm>>X*(Xr3zya9M5@^bK`)WT_;M?uTFT+KpLoJS0`Y1+e{(m_ivrs zee}VegU^nRd=>tqyAB;I*w``m@?~=lEN>jyu6ym|Nq5(9MM-CC`|8Vj7N6eL5^^E+ z-=91F1UV;~ceEC-nHN|ww~3t*_xuL4<9Sox~SvuvZ{Sc z>(_2MNoO1>*Uoqo&Um}#ChNa1Pnb9PjBLJ=v=?X0<1PtUoDf90@FhuFngkr6is4FQ zDws7Mv`1l!0}qmx6@lw0ehMqnZ{viLnJgH(q z+M>j`Ir3Y9grNt@V-|F$E*MR$k;_;&(Heh?Gg2PYx{C_G%`TL`XiJM<(Jw!M57`+o zcjN=~9M0nE12r`FE&&EqR=Oi90xmV=-a{|bp?W|PsVgPVfw$rJq@j`OA@PgSqh0Q- zD7_VF$7+`gdT%sYMYV)-jtQ@goDAUu`Dc|Dd-;Nejqz3TPwV3cAMnR?bf+yCahPjq z0*QcZ|GKi~RCyCk94VAUkMO&!BKe3lJ$@x#+-yVf3;yBHjv-gmYh?bP-ctTopav-QjZ$;K~E~;UJH2gc#|a#p14FxMDwN&nUS`L z7Os-sfH7~dIP?%}s-31b3+MU?yF%{W{P2BBizb6~XvR&DHDTX#{`HDGIJyYI>f9W6 zdNf+NyRxDvEx;Rw2GkEW3=e!E^aUR9X_=0d7#FyD;zHPzOj$G-X;_Gie+_*xT;kPDq|7z5@*gxP|*D z=6)D(zy}S>$ZW`q!+ZrVZ{oDuKv}wuIzl0T1cfy;2aR-f5UttFoGTj_Sd4g*soxqQ zm`nq~J3ziMW4VHOC~W79n?SX*UO!}_a=sWGqL?c#D$L1BNp`2(?a}z-wFT?nQ8D9B z^MhE?L&K%Xo3IMdFu{~NnEUwSM6ZATbC6=g$(d%Bh-XHgXkI8kHu9Y?I;D=P7>pH0 z&Y%|+u2AWZOoH?28Q@IO8}y%<23`iv4A1NOa1d7m!~qOMKmTfziR%g5#hLy$d+z}s zS9RtM-*fMsxznp=MqQd2jYgwU@3L&^x_4VP2HW6RmYZAwo7e^%z@~0Ou|pC^Ev1dRV$Aifmrg9 zk7NXstjHqyNk$QS1}{#vzA$(&E0*Y$&~s!Lx^q?9DS0jKT$}J-wIl{knL^%vX~+*x z6->391QIeN4FX0fkHDuJ(U=*DrFI!fZFXcM@lqW;TNq5$C|Zb-B-)6=F>L($+zI#w z>i0;Jj-4S=%hLk4CoW4U(on)v&Pqh6Qr$g00mJK_&+c>}A%{Au-=eU*_PUq;ZRd$s zj&8Vhxa-jD^o3e_XZqfI?|pkKyGA?Se)w?fC7<8JUU^M>;n|JDo^P|o&)xIr;^x}% zopp+Bc-W@YNw4nsh`Ln!*Bx(E_t#&6?=iv}2*eV1 zHo{wqK~#~2(8bEuAVN14N&2{#G%Hl*Nko|zQn1lw_GwC)5R`uoLAkPLWqi02g7Wu< zsv@)Zoxktubd4+6>}m|DTNBrItsJZ3GBdjN4R$RKdvj#w*`*g;G}O7EvQgP(vu@jQ zq@<6e=P(S3w7(j}XZ!^BsY&<}s!+g30E?nfjY?7`+O(i0C+z*gF^UQW%19CG9RL2AgE%uY(Gy&Ci6xbqsgP72vB{dn=;*K2nyLR0ORuG`=J1mteJ=;>Kx>S z$tXmyX^?f#=)jjtgu2Nf3WmlGC5~JNsq&X!+WW3_yY0TwrTIdq<#jGfK5*}N^!Z0-V4u5Vmn2UVISGD<?dvD!;tWt>gWTdMBp~h#U~Qd0bH;nTMDHoY}~~I zd=N!0JMLWxpOGX}-W2eGF^(kJPK<4tnN}c^)rM&QY36}2n_pIZiqS&|E5}y>Gdyz! zbjp&5#PoiQ)1bp7BnD0tUD%O+GAV`>&=|x%>FAW60$HSdLbLj=14L;axL1dH(!iks zvLAE-=CGjv<`edBhb6@>?gpR{U4oi5I4-l@4ISN@{@z4MlA}<}0VbVkZC343WzhQblM%Z30&z4#G*O&a$OzBibtZZPIeIfW=$Z}BM#`mkodgamra zmh3mqcqZ3_%$d9r1soWS2rFo8Yb3?G2?}2-)Zpiekq5-i6gKnqXl`cFYj7Ztk}KSl z0p8o|hjgB&OXo`FqHX(V*MAW+oS;Kbl+Cqgp1f(k8wFiK_s+gyod>?rA-tq|OCg6f zRUn?yiR5;ZakS-Wp(q5v6vQbg%HC|15>N`8O&FzCbL6(rZNJS~-E$_Bp!Z}<6qAkz zcyqo4M52npE}m{oCTgq8%Oc^DfZyl!z_;4LI`n`CgulT0jr))<$C==c!1P0Lhtymb zxmHMy_~eOCugK>iy5ZuZ?2`TO+IRBpbr(KSw)4TwY~Dl`VpuJ-dyZea>l)FsXn1_@ z&QJ3^KfNxB9kG)&!lgQB8epiRL^C9-Qc#u^rM_@&aHT}_cmojasP(0*kPygbGL#fS zH2^AvP`)5aprJZf2(493eI^_46zwqUpP{M(MEvyjnwstGxgBv9G>e~Uzm4zURZ!lf zVrTS|_En?&*@JrdGa^#0@C#mnJv9pNQG*CBma3d0YE8gchY#(fGgG``J<3Et*cOiE z_8;6vzHu&RAf8=7rWn+?ggFoot}il}LN+{&DI$ykf1=LM3v5~9h~XkpAuoP~A8giV zj);~{#DKoNHgV*5_u-G12RE@7*9FVZKiqv)b7}+o_O?LTUD_3^#DVZ^Sf0dzAdJ7* zIXdZ#|6#*r@lx?)x`7H0PH)#~eI021Zy|msg~jTE3YOhEOsvM9X)CCH%y$FN01i9f z)S9d(!3W#HR@@--N_HzrJrakC^5{OC%!5wxw=l1+BC~My088~&^{$Mqsjlc}X&8*f z`WAKp%&HfSu=S;q)Yc|RrP_1d3sJb~gJmN9&dSGUV_l5$A}tQ?tG-4JH6*++t44i@ zC?QWYPEavVuz_k$Gl-2q&Ky}0XJ|w&WzR5~8H$Y;9j*u71Og?AmO4G~Cie*IlS-Qz zwKm!AEe`E1+{xoThd<_x)jZYIv~BYjPMv=kw#M`Z_R`k$-P+M%u|GV}^wE3SRqIEZ zUALBWG<8ZHNN-y>YN|-Wu+1vv5!?K}Rv!f{bGY1M@^@N5-QWvzw*J>+AYHLT>=hnt&-`tOw*!YD|2mKyqTR5vc zqCH&A>6Kh1{kbiyCzu-`GVjO~=E^ohY63_xg-rOk;7}V0TvN;;&~^~y^xjCuR2&O>BTY3E zE%~ONgtZBfD#W=dNzo=r zBe;UI()Jcnx>2te|D;*!oC=*+X(i&i^ljg|e}8LtlwBVde5d~x7V(gBTW3^ z-%QqeNxKo{+`sd_r?hMR?xlA7g-_r4g*O~xMM}4|bJ-HM!w+(trP^C3fU}*#HL5Qj z67AMjJ7hovB5^^5fEsf$g-l;oI~5v)kj?DknBkIzV@}7}RIx!g#r4_2@ZzEJL%I>& zOB1nZl!Fc3hpFz1iNN_~iE^VSqXoPu5|K}6v$yyIS_+`j=8XEiUjHb24(0dR%K`ru z_WTxKFz7>zGIUm9+qHYLrQ*dm9Vk{yyr{&XeUo)(e;fDGD8NR9O+&EE<*=E}`t{D> zp-PBxVu@Asu$Ed;up{f8OXAt*uAJwP#i@Hm6WWc#ukAVLMw@Ze{gD{*&*57GEM@zc zgRI-i2pn;WGN6!@M=i*kjXCnM8BXHpk4Y}s<&lpO#9boWNRtJZM|4qi`lMf{_RnyV zq6q7A)+*>-5^B3pgQ#nE*KA}E_5zC#C)?cQukgo%{BRA^FBiKFUJIcibC490msf?# zIDAIdtY~l4-#g(d{CD1z>ylF&{NP7Y^srX<@)Rl7&uU=o9i;D(1~Ki)Qyd5o$8gkDc}gQbPu3-PM71>U7c4Q z=r`ds$^9^4nJIbCF4k_(!YB`t&#ncoaY9YapDR{A~rgCA=NR5Bx&DZ zJ;TG=Hw>EdN8n(uuu!;D_0^Y(Zfgg#yN7U_Ggz25iLIL`vRm+oZcm0XdkMID%!E#w zJEt8@G&*M*C(`Bf=gywh(*^Z9-PnL2K?5gaK4NhY#|-!NOcAoJ2c~Mhq*c4WjYk$) z{`mZVX*)K9d~cdSzE1`Go7q2J><_GqKk@4qM{g`&-CgzWy)`U#_iZ}puF!5Of~c>t z+AN}Kf3V$o?~cPZZL>Uklx}3NwuAG#KE!-`JL*J-Pzqxtf(UszHh#r%yPb9goeFJ~ z?+>{Rw;Lw|GvtnpO_#c@sLhj&ENa38-|Oq`=|au3G>2s{ndE#=50Iz0Br8RH>r%l? zZn}H|r0}xmE-#%`D%)jGr}xOIqKr_wnIm7xckx4x0WljN!QBG5=P1>2oVMEb%Q?iBHf(TsLXqdp;L-`J#hC z8qIt9@qEBZo*WdLM0`MqFX7tRUs*55U7iQZ=!ynKo25$uk1zwj+F{QlMB|w&5VpN; zd}d=~F5)D-sPVu`{U!J7Me+FNJ&r$;7uQzT?_>Ypet9eTtspi z{rvILl^@vk(6so+CA9IUw(j3I`q3=RPu%@5(MoNnHOA$$^Mp^SC6Hd-S?P5?^YRb_ zy3*|h6woKSeHnTwcmQu9LsyQ?mqOX^sOqbW=@op5nUe74K2k2V4YQ^G( zCVPg3?2*{$hh>iigAA(M?2opr7%7X@aDj*->Y^gMh(?)dx2Ce8S4^Ag zEs+_$rD~eBQ26rnYas-ywjruf6s=e?)T4>}QWKL6Ahw%DJL0r9=dfOCb26j%hR*H1R1bI~~XOBQrx(iUVI z2|w@CEwDaqHl7im?LutNcVRRBE1nj=E`P<-45t32@st9zW`w7I<%@;<{(m`N6fT>< z7d!sa^fez_N%{VNg{tP!)&GtAoZ+pceGb9yNAhYLyI6HMM?-Gp4I)H`hxL#=QbmJq zgvnolK5Cm_>eGeRIJ^Ur#M`LSSU4y#ba&RqXY`^VzCP{yNPL~)`$&eJ_I)JC&hULC z&BpCa!Cn+- z3JG^uJCNzLJH40-H6427khatEFQl&s%h{7^xI#f)=A>7Y9O)qSL>VxG)d>|6@D;&u ziPDhVrvwwQ>A|8XK1!A0+M|vVY3{j#Qa~+sFoip?iccZbnDP4wMb7d4geo(BKcUPy zzMoJh4f7XF;Apamc!lYpq_!6w4gFexNq91s8y7Wwixv#csZW!hRMxlzyu0%;c(UquMrqQHlqF+s^KO6L@o1M}XW_Np{%8z*bZpsW? z0i}wx8U3M>eMtB-VI<+ngpq^^1tSXv@e8Yed9`V+LvA7>oF#T1Xxju_uQ5lN)<1=9 zE4UK0zUf~^3!cHtXbaQ7j8Jmgm#K;TwWGEl`c?q}5wU?=u25Sgg&E4Z>LY_VRC8cn zJXlK(-zc;jYbc+RWz1p}X@c%*g<&+LssD0xq~@yX&7;54v5P&mg|LmSBg`9RU(|7J z@aDrAdY0p}9Iql3Ui^91Bxb7Zqz{cB;(B{CVu;+0wWS`Qj)S9)?vG4jjKB_<4jdZMS8~HFPfUIUHgN6753SjU{QeY<oTBUN|apq%@ob7vcq?d0~NvN5n~ zW!p-b(gSjztfJ0w&9SJkfN*&$Z|kLZvZ5tU`#xGSl8R^YKHA8%@8crDOx}l}x}x8T z^po|*UaXnBvU z`pOaqPzPZpkO$3^!6}Y8$hD4?8peM@atMhMt5Xf}nBTYtJ}18g#F2PxeGa~dJ14sV z3)Wn$?U)q*#}CaA`$=T$64!Ej$p4`Ai)+te{fMKownHA9!#=OyUr&FsA_!?hoWre; z#sxN>TCz{+dQ2Qk_hI5#LjA(A<^%JUIk&iyC?ld^tVrC4Xg_AhKYEEyy?q*SnZ1p+G2Pqf0?*`axWCMTXv%{VH-K#L_2xb-rlDFb zL9r7Ae54UOs$b*0Kpx<|9(ksde1i*4^2}hCdL)Ro{cXnuK_6nE*i)lE{9#dQAh?-5 zxxxG`PQRe}reV!~(^*21Jm@KCFvY^(&3M}ki^Lsi*ScgsY}>uUN_AORW31E%LJP7? z*6p0_(^0!&RU?MEp8voP9k%-39%P2XBUumEI1HX(dp-|ccjKxX+st8h6Q5-F4ROGK zrq;aqPREy=uE-o#J*&iF{@5Q1nFi74gA{QZ)C~5Q|67(B`cSw4`;`2$AGXo|P5U%d z_jB6kfc6Zh8S~gH2F(~GhK%^(dAUsAqd&D~pK_WpVJxQ^6UK6yQ7~35cx4XFh;see zWfNsYswgOSW3r7x?vL7)S#F&(9sMBYJ2;2;($;45ULqzlelL;_rra4x z1?C2`VnLW3*1bJ#Eji*rHWDf$9!x?>CZjmnTL%A_wS8CHl?P7WI#UXhD?AjEmBQZ( z$P7cJa6!Hbe5_8m@Nt9#0}^n6wgG_8}#Qsy-Mj z>V&!oqTA8Y(T>n)Dssa^!@82uqD~h1Dy@5)KUk-|Hy+=LAJLsvUa~bFAEpS%pi_Lk zQj#jiE6g2;yiqH{xhu#0mCy@7N9G|G@-g%t?G%owVU&xvSmAEBS@d909|R#Y3?!Nx zptx@~{&xCH zG>Tx;Ci$f&tV@v%jvUxDh0nGUM8p-j+r+-$zh=TVoV{%XDK|HZt5UojH?kU&{y1%p zvK5TeEC|HNATdHGGe48VS9(HwY=zsaEpvNUXn(iRiA)~u*xk5{f>!wagt-UX7s z+rs>IHb;BT?q`;6{+oEBRFr#qWK{ zKg#WV7wh*B*)jSCczCl_8Nue1Z7Xwjp{;Ab-bR^1S2H&?n@sd=)@eQ+_u?mP8MPw#y;Z25FLsam zE}ulvyjgTi)O?R)t~mxZVDjHnZ;n9?n42bkm$r`Rh(3mDz*y?_ zb!xkEpF4&#&pY#;-zM$H#>5&E4#B4`0pZs+gk_70jVKaP+h)P#Wp2sk zrf4)6>FpG&3#!IOC|GyPN_3pHDI*?=zaB;WwVK9bKEg!}e+uX3jNYz*B3JVO=-w_Q zPn$8P{?Y)2oCZpLMy6PS`6m&W?5Xe zV>6wmHcd4nLqhpCx+$T#ysN8=!@ht|c;6wPV5t2!GhQ)XJ(dsE8Jz^{7H=BA0&E}& z`WpTV($|uxPG2)B$PDRgSbKGDSbHQq5C4{w%w>srVYV)ef7M7tHR&r<2^1TFk2^zO zpL}uxeHC-|)GCasoBhb-XUPgtaEdFSTaopPg1#ve!RwK|V}eI;3LfVeFRPMUU&UwE zSK-OM$op?hq4IXLwnk?nC}rda8$pegjvT9EKIiD?C-PBkRN2U88%5Z=POzHvDe|!y z&7mT(^bPVc1IUm(sDiitbva+K-DXxbnj_4aJGU`hU(nQ2$nj{Gg|JzywYcu>*v24L=h;BqT&mx6*g zKq&-s^XP5>g$zMkRuU231_U-@l}GuHkeBpI`#L7i!64ZsyIp8duE;Ls>d9YK>?P_z z;e68-;Wn}-X<3=s+p|lc+Wp`TSWLfwY>>`&;t!lB`%8i`FP)j+M-|b|A)|&{Y>|hf zT&b=7;@66v6^`^ho6kFZ@@?rq#`kK+p8Oh9r#&cs0nU8*BUfJe5%!;ph6&H%jrk1D zDIt7D4fv7z;f8wvnh_OH;c?9rPnZ%cR7AyM8$)EK9x@IbR}@3HqyvZVij8K<&LlGv zkRPw0CQ;tRoZ)a3LoG#?pon7H#&?)g%-3#5%CTN-CLz=Tto%yo#b$~Ik)v=fY68S5 zy^DRSP#0r+Jz*ByxiiQjZt;WT|5Zc_OIA2eYMm#XI9=)$@eA~jXol{!TG*W3e1wx@6J9d2@Sv(kWEiPuQ%bQ0}M(u#yZ?I!l>B`t4Bk}?z^C7<82Qt(_XBJEc(K%6Gto3sjJ^B2egZ|o>;X~ zR>8`gt>98Fx683|&m|jzRa^FNnJZFARr_H&<@ zUtLwk?18}G$5&kS*#K&7F4A^z+UiFH_er&~2UXUstx;(C1GN<}088=^@Q~`Z5``f7 zuHKaE^xhi)sLiOlg6aeCby4Mo0qPCjH0eO(hLC*~*q&FW1vIt~jZ`|@fl*~8el&3< ztv}dHr6kN88M=#>tg7r0lXtZFCG2XB~U$Lkf%Vn znB0M+U)^$|TZPDAD>LpNl%SkQ5^l3zs8lV*ui)>flSr(c55q@X>vRN4lYwx`M~U)g z6a>e^+~N(~5RFRZdBq#(npz~ywmn$de#`r3tvY#8Y48&E*7o%JwarJbXh?5oZ(R~B z#WOH$G{k$mdR}YK^@rG_{xW6*G?Y5+$mq(t&Jg=1DzD)IEAyXz(pN@S5o7Q2D}3)` zYFV@z+5afXDJ77g-O&;Xpo$*&ifJ}N2QF04C5I;!x8U3+C_`*1z*>Ll1-#yn>7-;iE7it^y`vHXhW*bykZ0$3XAd2Yu zK)kyi-8_Wm1;_iz(!SBQC{Zv*et?qHrt_h zS3lQ>vt%1(u*d;436aHM*YUn!;=Mx=6_ylv5epAs+9N|Eu=$9e~@Awu$3k59)50)yE<0)KIH!u(i(eNHCxvK`m1d z5he5q$xv4bHS4IQ7fjRQZc=K=^R6cFlZa<}@x$ye*m0?rorqL5E2XMAI4;KgiXE0& zif%I#cV~=|@+-TZeJDO1N^a2PfBwcfq9yJm*ZHmY0WUjox&Du?^E*ibEU@GfR(1$Z zR1R`ljtOpP@$Q2zhiIijK(fVcX#c3yU0~YP`9f#@Oxv-UOjY}5JPT))Zzo356$}9b zU9;de=q@k?P{OK77N-}~R%T`@D!Q?Lgr?i)=z zAM|$gJ=9Huz|@R2kg74LZBvg`6x;foIW|;9Fie7ZY3Ny#Ah40@lby3kkSC$fz$hqI zy?E`5Q{Y_|c&0pKXeZhuY~2MHXpaycayp?@fll-bi-o(?NF^$mAs5|c>0mY&iGV>c zCQ}S-G^IOk6?6%1(dFjWL81>HMA8}VE$;itWW5#}OqV-5iRYr1E}AzuySJMVv$4VS z1;xn5$4caLxZ4~pIV%bqx#-|qJQ-c7(te};UaNU`I5hsp37F~$h2Jg4-Wv{eEULcq zjq85-H=hg*w#`d^_QUMHm%pap_{8|DMbwog>`BvURO!+7SS=TQ@%GQ%4mNhH{0Z~U z>vA{U`O=9t4N6WoL0vzorGlJIjHCo^u;97rO_L5RQeCsas)1MzlDIQgja;F72Kq9$ zYP{#0Ins7fUD>c}Jd`JAi;Je!mkqd`{fW&9994m#Wu`4dk>?=Ty2lP0Xc{v}U}`Zf z2Na*`W{QXEDWLRH#&|H0H6HLxYgA~>rnBcy&=aj`CYd{Unsw>WAkwV+3hfL8#dl_B z$VR^@@77j*@ykU)2>N+&^Le{XFQ#||jbx6Wv(dq8qG$!(eQl?5jLQh;313jFO?v7T zmN3birKx=&q9nl*csop5ij?Txlu>Q>q9TPsQw6sl<|Wq--F4zK3*D%FQ%%BP8tJw{ z9Co{0?u~-m?Hc7Tf}9oWM%kH_%NEX?-QRUy_jwaMFM)STX7lPY>5^@Cj_{gCfY+Kp zhXSdqlaGRQh^JZ(wgszJR^0c8n@GN0Ixx6~OR|5f{oozZ<(+ohQc2qXt0%eiyHyu) zf1&;8{lwVGcXun!DveIo)g{8IB_`HwO9DV zJ^n()ip%rG3_M(;ymkCPAis=4zW$81m~sWzxlbO%38yJ^2A@$5srQ*GN2ab~u4jdXnT z9xfL?qE-*|i56Q|t!THXsDRr*id1VURBBU?q+6JP;%KqkP(B%cZ5gF-1%|k9D3DaQ zL>i&EeX6ep?Xh4SKF5aznjoJF(7^VyaNewbssxjWWmLiCY`Jr0s^HYX%eh?gZwhY? zG(rQHwDvq)lHUX_W$%@ie@F5(#3>QY)EM>a#9^-ETeq-V4K{^faN)TZ7P;Fc!jfa$ zrwWW@D4GwGj10RNN>C!gGh?x+o$x6xDSGPYV<>!5b<)kMKmviyUJdOelR#j`@|ebo zywFJ_WWdl*EVpW(x@2GUf#|*i+Qa${6KzKbN@A)c5(-}-I`sRt@u}Nye`+JYSBAQ* z-?69N)zxlUHI(U`TbFIsw4m)6eYO$aiUIq^O%>o_AjpSYSD0=``iHmx* z6F;RK1D+Ow9|#V{wd2spT)T4s0oKa)OkE67RJOozvuY*J;5YQ8mAvLN5%!wQKyqfFzDVJ)sYeX!#oyD7em|lH zBfWl~!@-RrFbGp-5;z@EAeivNov@c0PvZz7Aln^1wY)rjbMxr@fs5LH(RR_m{4LGn zx8T={14@*lL23_ zu_k!+ehCsSA~xK|_E?YbH4dr+sH!iQgVLijAsO3o1jk^K$6tOD5R_n>$EVcPjyyXS zaWxuOW0kzAonD}~i`-&)(B-R)(<`K|=>9+CT zmx>g6h;m2Vwjp+-ONBBAy@eX!at8AgQA|0{_P_t#kwonowPh_5NE1 z7dBgdEY{1!-W2;uX&Zlz;9cN<_W6DK^Rw-Eem6ZAUgqQPFMMvi$9(<<#@}FhF7y6R z8t(`A#4*dAnfE+e_}qBUgy+WheyhlMB$GACgv=c`lA!2Anw!q31f>^yem> zw6+T+2=)Gm>V?o{5zsOPE+o_u2WzF^j3~P6kiYDhK_y89-G-hs2B{q7ZF|Y9ScIyM zd>&9Ynv$lBN-6zWW+rNMB?vbwAwe^Z;`JCZvX64Y= z6Z=2dwlcN8vCV6%i_~|mdb?^Oojv=Cc0l}?9;@R6SL160iJnV$froj)!_v&9df?l$ zCE?q186_mCcyAXKutFOb91c7vi2Xw6mrZSCC_;L zb{V%1*&p#(w3eC>`s3hb)Q2Ju$nmTXmV_I`*-!(H&a8BEHdE842gi zHtj1$Zw9H6&D+U3m#oqNRlqg@8Q~HxBgBLoRcAFYy~GcGO2DP?Msy*fxFDc9N=aHF zDp2)KYJdXgs(`}wBr%|FJ&6oZ5g{{FsI3#HNHr=&QqcG*R%1F_;1+o|7Cr{J(U9pp}jVrba~_zAp=m0>5Ys30Rs zMPmhBoM2r&E{>Z5c5^RIIGndX9=?zjRY6~)*~?QkXR^Cmu{| z{0Lbs0oo2=y) z6)|hn&k_2!44y-{C+r|iIukoWfhGmR@Ht>bAs_{~3L6z2R|q&tHmW=v1#!;>-2{Su zYxXn?=s2q@I9@stPQS6gh_Fv}1gVlH&dS8KS`Zfn!FgCW$OxHhEdmcZ$VQjd7a0Ph z)kd;t(?~}8Ag3dso+aGo!ivyKu{)nkvaf(~cOV~yoc@QrS;b{~y_6`*O)+K~mAQc}9{)a}FI|^oyojsNr0zWGZPRL5Qy1rSlKOJy zEh?D|=J4MX>zhj|edU5Tnrdp&>6)4*%P7((%Cx2FR?&}UN#kvmsZ?b(^=s4R>a-k% ztY*mb!b%bOTslpcKbZ4;MfSPEGS8C&e+~g+h5XOvJZtW}joEqD{h4`+a1lKxzIOHv z@e$mCpfK|H#L_4wm*l+|TW@nA0>^1dSm(pNQ6OH zV#?Ey@+k`;=UdscW&6^t>o$G7w?3RKmu_A(e{hst-q5NIbtif_e=%TEB9sbO%Tm zn@XJ8Q&<1!x?6v9>!IfkAAau8;pYz>dY*8g5dzZ_z`HuQ($7YUao@CU>4}p!-|{}? zy3joBWy%Kg{={iomk4LwN>Nx;trENyL; zz*kosUw!_q=Px?6c=4e{d+uCxnA3{|*z6YMpIXr&m*`I3?}jXrvV0!0$m+-4bg)Dq z$OIpP!3De-nmGCSvObK8+~<&6Hkuy`O$P1Xidm!cHRZJw$T1U#C@+H9|rC za0z0$8}ScU4~U8NvynBG9?u#hZ#|o-Ze*wH z@}dYH9ukJ0@Vgyg!e(IwKc`1}6a&D;>nRjGqmcQ>lcj#244T9B5W0kUD-lJ#TEaH= z&8=}Bh?}TR`+vYJJd$);BOIz)VGf#1LV{^;iuXrO08nChCv) zWFpVH;GDsIN^)n*klaoPj^xYmJjn->xQ~4`Ahx%Q0d2r1T43)24-%i?woBkGY?k*+ z*XX&`Ir-O7jJZx}fc~J5dDI+}wQCB6W4<-X7)YSv$9c}U{+>4{jyd~#jCowCF~3WA zr?JA$H4w-z|@2n=pyuD>21qRU{p=QR-_tK zg6$(`14MyLK2Zv+;6^o`slmvmv}+~BS#a%ytgxuB1`{CpwGfSgm!V}u=5sQts?IR= z)(Ir~ZPcf`#5&Qanz=7|Q31f4d!IacU^Hu772h@fiz_L_=1Q^hDsuE*CBJ^?W3aJq zZ_n6R^>D@9kYuBd1#s;5u(dwUbMpZ+Zd1KRr;(7EiDq(G3JPqe=JPA^o}ZNZjq%Om zTRG!DAn3j3TsZ4H;a9JLU)>Ra7HjHx9F&N+GqgK6J5h@P?$9kft2_(J8cKgdf`oN< zcCe}7i5!@onraxw{3R$u=;M7%^O-Ff#j=rgkh8@*Ujvvp*na$D6L7DI_l&=pjph`+ zyK?J(Xr0&wBx*sd0SyX*60dn2mLRBp22zxfM+%$Kg`<38ROBrjq;RJo&4Ej_dsOlVF2HqwRsW4eq zp9ZPo;a>!8D9aV)VBPqiKj`C4RC5?Nx3gs^W~h^Ges_4T_J$h1?6R=R%I1dOEw}wF zAWHT1k{I||^9wK+CX>jhet{^XP7~e-s4aaw2t_{wwGdA@1nWTmZk=c7$Gy~}Xa-x)zsQGYb1Kugq<6r$p{S)$5Pb;ao;M=CAVEUCRxZK*sWzxN6})Lz{l ze_DHrEq*#K-w}VBEv9q>qLF8RVjaewje9LLi>FMP2&Rkd^QP=`$u1Bxz)Po8-lg#0ri zd-Ci(RHlS*r~m9vq$PMysjx$Jc)(U26n13Xx>{&tJcR8SgmDOJFu!xd50dOLc{izV z)f0w5@rBX{j}qy3x&~J&l-30zT!TZK#(1nnqzDN@EDD4mr9@g%ee`|Zt(#Z3hQi0$ z4Xul!4GYScXZMY2$>246rN@C?jt!@uT0RH*<;o$@icY}gxaAtIlW3|V%xqjAp_Nma z<#x!{7W5Uh!eGLCK|$I^q8JjJa~eVMpK&7uM0}D3B8rJE#U|z|GEQgCED~f};dXFX z>?mAqg=TGJZu0LdG7T=NVlxEl*p;II>0joA0t9bGfK;G{dcF5vNifc1$`QASTv`#i zHN`?mqdkE-$`w<(MACwgLK`40D4-m_Mf=0opVj_!n@#&&D13DM^`pe=j*1odmj&D> zne8iIXTDp0tv+za9iLFuPuy|G1FHDw*R(G@`vmKJQgaxgrWAa-#8u5R7+JMTtNDA@ z@Yu83Q(u3Gg-)H)e*Ms=v|oSX6D$N!1Lr!}$1NwNFN0oPq&mD%`Y9@sPA&#ehGF=p zRLn7w`K1vnf}EsnglSduBaC0`%>n!)9wNwtzdU$%M2LVn#r=x20=fY*jd+wPU1?lz zUNpG3*K!g`l~rD_=yH*YP>JPUu^0IIP<;l93#f7!yn0q9MC%S#13(3wDJVeF~ zR?(t+TUxySSnAMFY9LW^JW;;+{$94qi1Qdv^qy>;H4u;2)LPe;>|Z2?>f7`+qDDvs z_bd`m7FU%U#>Z!Bw*#}B_an)596gn5I%)QdSbG6 zXMgp`_3JlWxbb~U$~z+B?Pu0`f(s5ck3ZVCs@2)jv3!4$Z6LsREnTXu>c6>7J$}Ir ztJYpNN4DBrR%z{mRcF4j@Wg_y`72Z&7@O>oGFz|lI#rVjlyoH=r1k<5CJteb z5rc9drN0mk!5lcSy`g!D>iy^U&pNu;U+Wb|PW9MbO)ILjs}mhHN_B1f;~jhwx$^ni7Z$NJ%H+P69BjyaH-M2*Tk&dA(#WODA~sfr4uQ+IL?b+5_pYE5LK<)&;BZ z9)mvP`#?uPHwu5pdWc@Tgxg`)z4A%ET!q)MDK8HHjm zPC=LqT938k%rqe#v>nIdA*MuDh%4x#M8?J zFI+K}3)thw0opM+{#-tAS<{@Sfa8BK_{#ynH!3s>%hbiCxXIA2S$#F#Z%O}a5OeUk|d!Vmq3zl;86?U=rACn*g2{Mqw2g~e;_`|=O7U4Gs8r( zHiuAG4qX_^U`3$zu0k_wwjoDH;v}`Fodh;g-brXPI1=#Cs(><+%Q0S3(vDxP8#c7I zeyMrcsZ+P!y8fvTZnz-TqI|h&#RH$*^93!vti7={cvIOe?;qMJo>`FSzUp7!x4bRY zS|Z*M`Or;gN_SdQyPkbd?tYU3E@>q&(%dbVn<1`{fmmd_z<#$PY*7%1S3{ttxQ*2qe)xK^N}L#0O(%kXaB$ciPJ{{MdiOGZclv;D8-Z_ep z!x>hX$%jDPVM24dzOJS!TtaAOw+V3;w`b3z6P1uSe>e9Akp@x_r_PKVJ7^mmk9ovU za>LU4CfkEG3))_L4JWf{XG3hw&3pD<{XAQAple}#)w=~YFePr%?Hb~ze zOW=q$-7vfJ@UHo1_I~&o)HYjsVAk@(OZM!mBbvbk$JsaJA+ z-J2RnlC}@y;VD#jBihU~D#3$>v_)r8HSu%# z7OK*H?%zViy3gfX$jWf;-$Hb$1$HffK<{CtLk^jR#o?y57n?v_Ufu}6LHeY&$vUuQ ziA)9bNA)xzJ*B~4l0YS7Y7WM2pui&pMwl`*VHQd!1iB$ja^E2lE?&jwxYoN3O>e)v zHMwhdGPS6)=R=X0Gg9qS9R6+6q3zUIXFKnGD~aQeJ-A?A$zUqIY}r6r+sn*zy%J&M zYhxu2vqJzA0$M_ROSoQW5SMb9MN9+{9YBbr$M!&D0|Hs0sln<*BJhNL!AL{BZ&NU` z0M{%B;ccFF0wHOf)Vt~5?LF6b6GG4M-P8m3oYqaV`X`cb~&+b#&Ho+J%axjU{!6H?(I8v!d5cMsVi{Jt$`J8R(3TxNm7M&kcW9O2)6c$hM zvy0M-3Y8ux*#{e!KSm*a{SY%z+SGnbkU>`^tUn9J@u58*nzQ05#|{y-JpQDj&^ z2q0<54PvgF*@<)ZPJDsj5Uk8$HC|MFI2pf?VMM6ZbxeoYHWt6p;(M#09>E`3> zDL%&rlNJ_EMayYHt`WXVYFno!;&I5&{c*^V?QM;zhWaX`@k1&L27Jg;u?R7SVmje; zL^r)m#$@4VA~uYo2$`w~gQ8JF&ws^a4EG;&W-MKShMj|G*y)KNes^r#gCaJT@jp=*lXigQJHmr+=<*#b^FbXCd~gLx<*?L=E81JA$Ww(%kL7k4u`S@VzPk;fTe*4bdUpd5OzC1$jM%U zG!fZOL12@;LO;goXCvf@Z3Jd0f`KODWoHyHp&=jt;pR=Bh4(%IsUWmv00q$*^ zFeoajrv_Iw%^dbVNaYldM*>ee(bx zypwMwY5kplD@pC|fA5ZCHf^0edz9;v+73=@+S$icM-tLM_KAdJ9c$}=wJ@(C0W+gZ z;-2E~j){6?V-{15DTW?7=?uconZBJc^4#7|SUA(S6Xu=U+X>r{f|Jo1jqU0><0QLP z+lmf`ixTcu9yA4tNNky)n^6~CI!XnTb#`85i<^Gylh6!#(Cl=2qjcu7=PxgvRf>AR zo=)$PQ&Xsp(#~t6+K!AC!H$AgA~A88k4A~^%_G;J@gpzbO$uBe!t~L302!s`3lKAc`Jj2z z&L_NjXWvX%_UC*v;oLj>X2QTf=bH&1`(Wb+(9n}Q12m*`&IIN&nu_q#jc~&<1^?C0 zOJNx-4fOX?o4^EufXTF>*VUVWNdsoGZkWk3Ru0%SDRx|%GXWcN&8Liqe=tbvgF&t3yz z_Mfo^LhnC&4TR%!a>%FwO@V+p!xK^GPB%Zta3@GEqRTF))3s4>xtybP))|ZsFJ7n) z0_$6I%z4zj?_A7z)AeWHsLQ6WPs;yS>ejxSr)kn|%Bn^55icnnMVqwi`sfzik2r3~ z)v62G;X-*B`hk^bJe^3Ke3^>N`nsk>6TUXyf#AR<2a;)nC}`e0D8Y!7%_JdNpO>Ge!gYwiiuyztog9@L zw>6d|YXXWrxKBE=CFo1}mu~&UmD(fkyZr_BZ`zlaH3h4+XV`oty>UhS#23G_&01!i zGshaXpLsq|110Rscb__SH+ulhNB>28U|Bq&(~1E4jsG0B(kS9vR;kOW<3cGFlL0Jd zyuEl#5E6B<#&{!M6Q#X1I;_|mDopfp16lsw^uFTQTQA}txi2+PvArQsTOwOsi^aKv zZbyZ)Ro%P##}|$qXM^8ut8@p)4?UywEU3En)N0u)*VV~xtL8%>zx>RmZQC}TW2?PT*nNcfAY^^Gy9*tA93wW_d~XmzvAkDFRs!({QncK{$=+wQ(Q&UD(xAY1+ff7 zSKEaB(j&UI6&rA%Tg1BadQsoxB8FsSX0>nY0L7&-?A!@C^*77{C_80pTAB-vq# zkvZ=qF=2|4IqxJ{V2Y7B?O0px4mzOvVE}a9V3OSc zg`r9}qDTcSX`suy1)H}7<>Zmd=W_=oVlE-S2Sr4Q8R_s_IMWv(2P!*#r4QYp@?SYzTh?DL13J?rZ-Xs!AjBK8KIc6i8IU91!(94GyT-g=uK@6!&#mnN=rRC+Dx|VI3_jDv2DfhL+>r>(1AJKmBgHSw% zmh?eeHfYPZT}ga1t3zp@@|CCRyEd$CZ0=jyvY>ZEWcm3G<%`ReXsvdib$Ea+ zstsxXz%z5Xjp^(g);7qIL&DE=6LsZ4UkmiqIbEGezXIz3#RLy|UC1r8$nbISkF7;M zHmuxKc>;zx0ak7*ctimUKq#Rw!hw)51!uG2-Zy9Hwwj3(V7ts8iy;)_el^K^JSIRl z2IhK@P+V4b=~w1dIZaBE72+uscVG<8&?&t8kw|Zkp#(D)v9@VT7J8i4$6i#d?lJAk zLoA{F`lU&g0Lx;J+j{?JZ8pyacIn01Z&~Xl!)V!qMg`j2cEr*T5Bu0v?5<+ku>;Mt zUVq(=M2_L%fOfyO&_}t#$m5V6;km;UK+1%Aaa`Pm{MQPhQ*8^8#LVT4o{CIkacWSt1*tlWag9Sh~9>v;!!5r*A?md8BZ+ZNg`OIRAhyGF_` zWzX)c_*7X*7;Ukh3V81LuqXrwnl{O=Ii7eGU@_qIN1i_lI3Z8-lkz|2=U!sH1k0Zy zmnR_{_P~p=LsxkrhaD%z^t*VMFyK z>Yyd~gYl9&BPozV#K@kK-E%jx>+(oZr}Wd8*MIfFc}Le;k2YKTYi>ELz06X`_z|m) zgu*Y2_uTRD|GlxIN@Sbgt_i>L@+ry@L}CVY!Ky;1aPn0}u*ZeiV*?_rkEvx|#DqEF zerigfuUagMa?3gCf8Y6!6bp{07TCEtX10-;5|8ipBJGvihiAW5o#2l_7lquz=?L@Hp(Yr zQVjwHO`8F-6c~C^kzLqH zQnL1HyT@N2d;fkcmSXrm09w{497GW+KkP76oSuM0_asXFwveQX!Xt3VO5B!$`aIx9 z{G#weEt8Ew{lc6vJXp3aUQu#q*|^m~Gk%uJ+NCeM`ub9g->9+D=4Yki=5 zT~G1>Uy|a?>NZN(P)aT@9H2ek)tcNjS~j;mzIb+XS=$ep`+Br9ASzio?$?{NvmnZf z!-{hCFoMxQD;(%^KzNZ523TbIW6jv8*NgauHU`VN} zu*Agp1dgJ~EP_&q0-}5W&`rt)CbYGI>vXqucXl+U zh`vN?a}KYSwN4C7C_KE0)P?cjgresb4&{;V~g;>o_%1PQwPoo6>@BdEus(-&}=Fy1Sap??baKK6$>Uo z@0(08iSY(x^HTiCbo2A#P$r$KpB60ur$zm9@=t4U4$|=F%~>|MthWR8C^?&~Fs~Xq zJQh8zSr*hKBcYXh<-tK@)2QFpD8~8aLq;2q?xu#2ZQ;VxyYq`z?mYIjkKG&&ZAstx ztLwGjqUBcgRpHS2M-Rt{Q^?ZeAAI<_(5$AuqGzdH<=FAn+j-Zm9Hw#ICqDeSPokyP z<+y#|ALwJN^Vlz|ju`y*NTc9>dm@$zh+X-9d$^c+)5`+KJ*h|O9Jfe;D|_OUyx8ab z(Gz{yd^$z-pv@*)Q$v|XkImAOfZGL;4J?_r*|gSke>OP-4dMo&HHSMHx5(tsR;yH- zZE>p;w#)OG^i6U<`18?kX*)A*ZRPt6&RdjCe-iDvI9zd0{)z61NzrgiY!do}+3Mi_ zG;WE_nSEf~k}i{)39=KnM6YHHAqB0g0eesq2lCtm_`9uE^JkvpPA4#n?upZx4Z1>p zg3~5DVH%nC4LVhuW-^brvcjiMeLVHAxAJhS8Wb8r<8WLM1k6Qiawr<^h73IvC+RZEBvYunT!vZl(PVPM->WH!QIERhijPiA zbbW{Sts0os)17LFMN1>340BmhWh(Wp%8zx;C8Uuc%dt7>B7YK9o--yZUCb;zPw5|a zl_|>Gti&1c?q+{`y>>nOz%}E4JnReDwLfUjD@xf>ZRJsRk9MQ>yP{f=gH^h;YuHhg z5b3VM?G`Z}r5rNdw@vDpW9h{Q8p?WFtIJU(u=@y_b%rn8 zy>$1E%JCP=x9?u|Yw0!qfz>I&&PfZlWZDo;N|_)Ysi zy0fEW=STO8F8Ma?O>L+4yM?j2RD*{6Q=1%ZsO3Vi?iCKR?K-?`7tRZy?99714LW6; zZ~(I6p?&=kxbSg#4A|!lQL;lk<`Cl?W^q%#;I`RqZu_yU2Zv4tX?lo>wfS@(&K?h9 z6u1w@n{nX~&s>pl22lLL#5d`_g`D@8&ISIOm4e4Zo*eU?IgW~%zs>{qg5Z&55B_Vu z%|p%wwWF`EtBa1j?{MGYOE1~GXZNlh+t;7JeA$BeUA^pkZ6+MaQlGgabstrBIZ(cZmUs&o(X}yjO^VqBLsHCObl^`Ib zpE99J>k^-1UlyXma&-x8$kv*w^02JfqQEjJJVsGx8xrqritPq4RpzA+@$x+RZ<=5EQbd%Sn`JQ~KAvnE|Nhe!55w=uDeW}{ONad*9aQ9Y zITALf|Wt+V{fKsJo#UQ+)y!0m6HtdQOhBO;4 zflVBP_P#aSB7~uZ1!J%^>bcUoIWh&_wx3^yVo{`aX`U``-ubmYRrZ;G< z+tYV5>$(&Q132MQy6rmcO*Xo(=JKx{S>Msz8M-TS`$ILu?6~&XMoF)CFTb>4^_e7F zXD?$gV0yHlXo&5^y@$faVH$kNAj=ZO1g5AX^ zDFX@rKvT(zTFoLrE6dwZE>jRJBe=m9%P6m|PFCI)NRuLpu-H+AWx~`+W}VZ7?G|cg zR2D)Hd;UD$Q;J&{3KN z;BW@Sa_1P;dP4c=9C)ULm(@~&Xrhi_YL#T9Z&^89O-%hC4cGd5L8wpF!>V!aaP3Z_ zz1noJoyGoDJySOm_`YW;dw7ZHA|Q)8T>>2-oP9(sC9ansl4^kgBaX0p&ihOVClQ;E z;&?Fa8N3hX0U#`OVIl_|XMadZps*RjSF-^JaZGc<#IZ9NM#U{rkc*=t?!J!-a>2|t z%~h8e@>AMP)SLX4pQqIhvVYB>UNPikFKZhYvWMZH`weh%xdE?FszHJmsy=Xd8DfHd z$^UnOiX{^fxo2unoqjPWFAGf%DrM~ZyKsHMW1t<#oq$yfWRno8&Pz>=$vA2Qx457S zTfmnMf>;I}f|m>g0|z=$(>Czv97HNZ!$tHIFh}JMP>Xz~ql`UVkqQUQf~bTqhQ>-( zC|+1Tr!Yz$o9NV@)Bf{NY4Ad@hg9>nj~(2&zOlvnrREiB)@pNS*#w7&uT1y%H%OgZ zzWRwBX7!g_PL6+ttwqEHg;r=k?dR&!Bm<4Q4`5eRn4g!6M zSV$XK2=B}QUxo$AniKd3!rY00*j@0#7#xJYhWw+%LC~zBU?#DTNhZ-XCPP>8Njj%8 zzKD7Rc9QOa|)Za$TtU94993=S2)Haeq9=YD4ux zSV8q?Ii1Vf+|r1l$VFtAck zWW@JTW|BoO!Ua@<1SDHl157a?#sxorB`NL$Ga*6X7V%xb`C*89UDvlfg5i%*uM6RB&@eKw49nFkJqu#qlvLsj8a+BnOE!lFnu`$J%P)xv- z5JDUR=~+S&!h%B}34xGGLY8IoX0s_QnW!Ufgef#^p z6kEFU+&lL<&w0*s%Kw}r^a*z)yv#eKN_lnEG8_k61EO^uV zW&h0dC2kmgUcC8D13HR`y$kY7pkuc%#FlC0tpgU6bYK9=S&@2Zl-eLOyBdH%P!NyE zd}f*R8MFpF=v@m!$#n2T@LG=}dkr0c(CmwRBSt}@@@5G6?aU)MMYkbSP&?};>C_VL zqNQ!+Uj@YhUtu|gue3-OE5BPne=&4}T=**!-Alg@kx7*LOTDpR;8i_}dO)Btfw3L{U|XrP~74 zqGO3Sp%)a+r_{ZI1-q^QOxq*$w6y{VUt8rwg?@CL0O3nhgePdB=8?fsy+q2lgWzBI z!UvdP!yo^+`0xJKr$?t6M@o%-aycG@>ybTnRIB7GAV zWv=0&=Ei756J~CBKD4w^2^362|5L4lGO5u-T~x2kp>eKnUmjGy|G)OQS|W3GfYR22 zJ&IMX2o(o{TcHP(RDQLf2%RtLD#9^M6L3GxTn2e@T|mr3`0;DTd}B%@2z3{y5&tQj zpk3W72SDo);e82TXGbw~f_^8?S0gav@Yh0H)=8|ZNJe)2G3&)2LSOU)IwdFD*ef1erY)+$0Vmjl3@Z}yYGbz}2`ee1-ik;9`M??qIZ+*MX? zgvC&`d9{DCZMbe_M{V_n$(Bg0(7`6(R2mUqV>aL|Q=g$)`b_YuEz-wegKQGsp9pj` zHC7e|@~jHhZy4*harKIXtgRqbvaHcog^Gt@1yLeHTR>Q*hWbI9_QQcHe~Bev^WRY2*RY~0V6O<2RQfwupu-OlAIc9zRy6d-Z}8d4>MjZ-=5Z7q!#)$~MG)mIeQ|1oV2*$Z@YsJbj= z4!J!)lp8|NdGek`kdDHaj%V{c&xN$Lket%TU?ctCwYb!^pz47CjcWlLj>|RWF)<=P zfVDB2zpu1#zlV1HOzOI2K>km@b`JGfE(d3XPbU03CdNm)I%8GkW)#;0I4Xd`JqVO* z>SrA!pv8oyuqbba4^NTeZ$_E31Np%9**Ds7N>j)Hij}S&H7|ApA4=owqOec9&%TpC z1>z8aRbfV$*}ZGS*s=j={XHnG<$=27@gPoskTkdZ=Bi3ukU^-uRUzqAO1B++agIDw zUwn%a^4Z7l$xeyPmSz7o8HZS5b&ah4g;h_( zjlou^zoiVo)>g_46z}2LfGza1`c<}E0cO3W1(}HI57i@pxTf^}TKIFr!nFxcc?qx+ zA>`n2D!^06i9mkv3aIxu$u%#oVJFogfzO=!QDp*O7ahd=v^m0+Wu@RzRKf)4``nx7Lifd2Dc^#}XhjveaS$TU{T6lN zgo%k3c(0`MYSa(1x0}bt^3+GvdK1kVU?bVe=RBHN1=EH=AR?x%m_Psnvhc8GG&jtD zABLM0W+Lx3I(|=(6Pr(8F|5m-2pwX%9%Wv;SHlY;cFZ9~NN|nD5Sk0^b2)dg&(b{9 zwL6@y2KDX(>U&PGb?PUslwNLDZ(-f(nI!YmGhutx8)+tNJNq_fLO$9-=w?N{4`>~x zMt-Q5b8{=xV>SXep^bTEp)u-rl_K7w5n~8j!6vlltNU}VkI6{tq-Vms%pSjyu48ZT z_W|~>7U+t1CJJe10jJ^=tukQXmh|Q#JDvC{220#xw8AL@zbExvhKYj)nv}*kH44d@ z&0bMl#m821DbKkpGm`U|i)w7u*7)4zt^y}yVcvW$o8vH^yW?1SYq?USen7qD9>4GE z0i>7`tq{cH%BDJ<$TJ~r1w_-W) z62NN0SFYwIr!3M*@&aYks#UlrQJh7s*0^Z2v@|O)y}2~{pFl7$H@h^ViLT`9eWKw~ zX{Yw9z9II5Is9eM;w^&W=<)cJ z1iJ`tQEYyxWcu2r(?=}Th$Ei=cgTi1?8R#A`)d*f8h6!JaoVJJ;(s$`s$=P0cL|iC zLyqF`7bDAEe8au*YqwJ4T@uqc_;zC-J$(KM#1n)$?YVn_*KH6^B)nw`#C_0&EI|ik zz263jpCU&k7)2DQV4BIapE6POuD&=L(CnMzzUc7BxyVCZaO5k%FzNSX2F*~ud0r)* z{4h@gdN(z=qxiY5!-p2-8y=-s?yasfJyWt=eVDy)zxqD)^gUikq&IYlWVqVrUBy0D zI&sss$w~Ru>UytqyM2SY6+&imK81=TN=TQA@{_-1vyq)6)E5>q8@Z;C<&fFU!gGm& zq5`zBXJsA(!gWmF+g z2@}Jw1|Fxs@NMwAC!*R|kQYw}6;Uoh`!YHZQeQA*AVf@I9S*9x=DOy_2E2GJsO#sP zXUBfbIJ>ndDOV(PAe^pJWs#oL+?iS)B!upy5Ao|K{Vo|>L{EVs|P&or5T$Dy5jQyr|zYY`fcm@>B7+x<@b1t(}*yWd!WyaFS4B{erG*Hx*X-MW9xNMWtBvGl|L=68o8 z^~m5}{=&-Qd|zN?U&)Yv+#e~dX>6{qX^i(qM>@8&?6~bU-*?)il5n1@tXTam9{RR8lQ1}4fjmx`8}il%`3!FXYDv7;zjkUZD5z4@JA zk38A*_m>@(z6KEAOk>RpkA;BLhv6A`n3*!gQa6wU z(XdMP#)g2;?Q#_WuyhH>M#?;bvL=tgJoxf;9`|l_WIF-Sx3gz`{ zrKlfax7`?@&6ECPzFX@&qiIy2~8J!z0oEtr;)8Gg6Qi67Yf?2>dn3)|xMQlav zT?+qI1Ihsau*pC~JcSxKke*r|Fp&$hH)nw$>Mu4Jm1!qZx0oruYwzg}#yF_}1JD8@ z;OWI3^kCskjzdn0gW^&@qdw-pz5&S!$v2R!fII}zfaHalGgaYeMOS-xx@G0`>L0W> zSCp0RZRl8A$j0u=NgYTf9-tcRoOT7si4?7CwBr}0%Z+AuS<(R#oWpwM zSQI|kjWD?uNpH^?j>{GlQ-jlT{(Ue2CxFSKsJewm)%*LpyL7T=N7dgn>G$8f|B`7> zC&^o)KjcI?`1n_xkB3+{ak3lm!iIX$s^oIA6`b6uF((W-1^KpC#h_S^r}#RLVL1>h zL*Nfyk9mx}`tfQ!plkjE48YQBJL95Hpv}~seu6A1r?hui@*@(7OTK&Vc7vkf8Co_t z(BGT2?QDy0yL51U@omTSKJCh{-o*BsEmVJ0CIwH3?gbd(6v{F;Eny?d4~Ee6o%CVN zUf1MWLU=GsvRN09Y)u+Dvn89ABpZxz1DWs(F+!q!sW;6feZoaJhAgt{;omp8<+w%v zW+Yy6Jo&F;WC3yyQ5EbZkeiHs{G**A_1JqO!efarmup=<(Ij^w&~7vh!2O{I+AYj! z$BAMw>L&{Ntt-F^nVArU!@DwJ-H1NH|T@;Z{x7 zjSROmx%m-;pgSRm$i79c(?yOMXEO&jERs;C)T@uFKY3f#n$o$)P0c{$ZFZj-M^0xZ zO|B_aTD-Nsc^GHTy@Tun=pgdWe01^EQq~qYcg&O#`+jS{mOg$`X>a`p0&;bPpFjL; z*{kGX4OR(vCcKdl&X%6;Qk*TzAPHSCYf>Kklwqfb+|kOBht2mO3dU=3b)zXDyjOF9 zjwpjB!7{=kur2tD3Da58_V(7AcvTctjmrZC79<~|yvV8rqU{J<<|yXbmAMucD>74-~_Vc$QLX6Mte2&Wy z7o4+p@b50+@kC*?68ziWS7x^XTf70io~E~qua&8IgQkk2`J$+Zr-56O@_^CRLokWS3k)Yu7~_-nXeH2(m3shR8L+I4u(2b8a9 zy0Z8*dpG4xLY}$Q?VKO>3SUeV4)lp;%a+Nms;FW@P*Pz94jW2;a^DT@v^E=_VmysH z6DEdJTnkH@5SB#r5!$d~Vh5bep*S|?!-XL1gmXbZOgb-qBW|Jb4isCYyB$0ElfV{b znA2Wi@1B|Isg3KGk1QKZK$#+*aQl2XWxyD`ix|?a=PxV5ly9A7oQ8+14$i3G&0Smd z9ugAV#aB=^;xQF|WS|M0tz&<*JjR_0f(a2L}XmFfL`Kh@=TJ| zqA3AEv^YVEf&KvySV05E>J8nfJTU_iNvtR}_b8rhj|8 z_U-IC^()Iw-o1~uc}yz^gW)oYzlzn*K4)~1A1(7;W4W{-=Y^YpC8kMlH3M{9FQMZW zA;c678p|TYrM(Jnu9Y0@O5~tBgr-D&YTocBBf%>(cZ?GdPt#Tglqa&wHgr_S#0izl z)8fe&$F5Yryu#$!`$)UTyh3u+D`LJ;{7qHE>^F@0qLQNfgz+ng<$>;GKTF%$&-7gC zpXs^Ov^IQ<{LRJ2oc|fd+-UrG`Z=%6f6n=zsTL9SIMZW3C2jw&2et3|uLrg7%8b#! zYk@IXukRA?YR^Gzu~^)t@lfhI-J5fLf6n+ZX}dmNzpjthub=-Hb-#F<(WO5>YT?%* z=}PrEiO4h8X_FbtpK-ylLTmEnocGw6dJkk~?89EihNHz!S|>a(LAfi zjJmDZro06RWv-RMZ3z|MaLS1AxrCnJl=7F_kU}PRv?MhSBQO;HqkTD}aMJ1zT3~?Y zP|F7d8JQw3zQ;_~;QSpfxtqOG?lw;ZK7*#nSzBHD8GCX_y@xhQs~YF>r&Ktd$S(l) z8f8o*s|A@vq%qU%N`U_hSpSrgst-VCb0}t2}Z2&Y7OzmpUWrH5pN+rM5S9nLp@x?<1vvloY2V8;VC@ zo>BHXJB#> z2u_#z0|k$Hc5-SBc)u`;f0m-`g#oEqvT{0WY}T*~kDzcC z)xF^8b|0@Fk)2q|MOm|6c|e@{9*z#xeucgG2S4~I4wCPQA5Q)jB}b_6sTj10fxY6` z(7Ggnm^fBf9OZk*BYY@fhf5jh83GV1$dS_-tOAxVPy9sVR;IwQx?igqfZfQ`3a5FW zc5Rk);_+0^%-HN-j-^Gw!2DzwC*{5*O>$?2;wN17q# zn>;7_`1B~x|Ba9G>B=p8_hzqSnebG?UVy~8x$9ViN<0P_hR8HAz`KSeB^Z5bMNK9o z2xl!-3PGV2WI@r11@Fqbf$UXYyyYTewN-Xe(bZrPmYPx-d>lkEf3cC&kXmqW$lIK= z==!pM?|Wigoy;sedwB2L_OYL0>EE^QZ7I6^9%~;H9!+??kS!t2B0LDMC4xops*}tyGm~lBl!`lsGLmNg_|#D3Dk4IHOiZXr3y+#H5i@`l z@Om+qtHR}Kaz!nA4lWp;$C5(SqM!*KmZ;_i4;wX8L2+H7ecR>svWmKRzcT5+@}R+I zB&{kGj}OS>zAFzp9VNCS+J?9(uQ$ByHjzCK(rlog?~s!(a=MWSK#GgxWPgTiiV1fo z{9eRU3J@~`okUiNO<%>#JpF}~AY|ZUhx4hy)B~v`kO;nRvtZs$ur$Loqa1Oe{PAkA{0Pm6(Glvq8dh;l;ZMntcEe*z` z&iqv)vRy{WA8!eM?Xa&Nwwc`KWE--- z&;JPm<7zGsuSqzmu~dQI>BvVnViVR;7Zym3N6!NZ&VAwxiZp4bQCbrE^o2(XZqBEG zC~F*%{Q=(*PXSEi^eR?JdSi}*#u#naR=!=l`gg^yLy&o{z~jxo{3y#?`?z%b2a@-> z3q*pYi3M)&>Y=?KE!wuxF>NdZ2Z@3gf4q))+i}uPCN}eu z|2R@n&5sqKb`yYy`bmY<18^g<_C@&=?gTv;+!rW@E^onKGCAX93A!)}Q3U)x7Z?H0 zw1-1xLjmu=gwT=S3k)nGz8Q5}R}#FQm1S*rY;C{1DbnH*@69%OB+Iw0o?{1gC03UE zXJ4DECy3`?Iv*BiIS+<~LkX_~ltS+?snlkGe??kjF^j@VO)n~=?S~v3VB>j14C?5~ zD?LEtKi`rtS5}5xg%zatgtZ0{z=cNPOVQYl_eas1;v&hJ zqSfN;!?$d_wzzw`Y&?8wGs&C!oy{v|cGT6>ZK*LeG*_>xo4w}#j-Hi&I~v~p!!7bn zL+j4n>@VIp)lgL$+`O}41aBsu|0Vu-1ak}_s{E-$uoye75Rw_W7G*MGKd9SErIIMR zhL#1S0)|rp8nv_ls?;|hidu-%nDlPS_|Is$75Y9S12r{#?#RT&25H)^xyt2=_yR;9 z>J5>zIqO8Kkc3R!6s_77>Wqh0aUP%eNW)bFrS^lW^KyIf>eCzhw$(JpKg`S<`p3fR z-HygUWAkzGp@!vlbr%YwlHLM!k$qU{IgI=Cm zzG0mDDWQ`M;(S3$g)NLqt1O=Ik4g2$>by)+tYZFLkuEY3GetG;i~OJ}Xj$ z!Tw>di$3h1N?;bR2Nq4&1|wl?Akf92fEBnxNINJzSA-k~-`$EnI|^FxilQgC@Q(!Q@J^IdPnxIDJxy` zkiKyy)hF~5u#C<`Hk7+}DvRSasqK?%4_!Uq9wKk<1#K7}pQ_Z)6_mWmvV#EYBb|;L zO@aiSZUIZj{8lU-+OqILNtCPY5w1+wk!UYLK{FZ2qM`}?F!yOnwfulrXX!j-vhLRux9$>u1a+)!Drw-} zP2Jal`~ZirAz{gbSVA!p{@zsACIFhE`G|BX?UKQ{CP{lJ9$1k23>R+Ad=8^oceUcx z0Kzju)*wR}1!;gpi?*^?kJ!pSjLI{`_M_6vV}@e;xtXDLO1-_Rbyx+4D<7|kdU+BOFHx=x>Dwm5jpQ#d9OJa6Nh*;*!B&Uei(GQqKvp~I zAmDJLn-xkT*WZ+u{*vzCJfRu#)YJuq5CR^}A!fA}r&&a|{V%EYTo;y#{NkvOAnQMfqtBY!5ixS+jO6Bs zkhUzsZbhYu(?aT=`J0WbzkoPU$S{P5e>-As( z<+$WT54_|Q5#N{bx}AG0^K2dL0fIj)2^%jlGali!rd4}Z>!w({`8incTQYB z+FksKNPEc~-CvBd{Fbhbk%m6&J#{CswvZR-Pjn03vzsLncB)GeV-{0BQo+b`ML?>4 zD1FeDu^G*%nP7%e1EU?gkxWydHcZn1nI9T8Du4vDDeg1nKMT9y0;Be9*qyY^5(_*A zHBoN7l;;rNbm_;T37U82h4_=wFTi5PBqj)%Lx}2 zPziRs&1!+;QUdL|^R&iu(Yf4|Zid$a0E&JJ!=?*S^&_se`TRZtK!~&5mse-maFP9i z(rrDHQ{Bhb9Xs*$C$`>QddT4$Z`ibpn@ukN+;a$C5D|#ZWS8l)a zfpYt+_c%*Nd+HLsRrM!l-uEb$8f#sPli&^LC!ONc+FDo5S?l%%iA$?(5-etux@m)%F-F8pk4)bM{y#>&!Fw z;?$WUy~p&}MAjZ#aEoa>jLk-u2Q#7Bz>ve!9qsHNA6+?m!Ve27t^fUL*V018l$C$^%zv?N%B@67j> z^XNpmgkpeAeNt<)2VGQIEAuM&Kb(aiV`c1#j-zKb$MQbXee)0AuLh;!*j^US1f*yg4azCa0z96%jrb$=MVEAgreGq_i{v9xG^m_< z%JSf9CGvqaoe~aXh8kM(&U62wD?;MulHY|+S;D+Wlap2KdG#In{?u`ld~MV0%N6+# zibd~No5V8+kcl{VoZ>HV?ttEsOC#Vy^wtiMIiC>G^}}J#Cwz@GgJI4m`aLAi=X|2y zL!x@lC;B}kL+5;=-$T-I&L{kyxff}8N}OqPjpwW*XBRlX46<<__yp%Ns-bD@0yu|$ zk~ZYrg7mTDsg>97Bo8I8{ZFf}@cZceyu`1fGxN=S6;bvQzluoyX1>aO>0d=_Sd&9Q zY8`!wJdTSJomRIo=boJXwiwkf?R!#e0S{+-+mVMe`#{Utc2E(;UnEDha69{kdN0ef zxS4H*I#S50{%XmmvLYJ1NPWIZE%unF+0Zui59}_AbWvZ7xO_L{T$k|Ygr^aLt&Pbh zcVtGa;FdIe9R6l<_Cu>xZUoCAeN8+CX#?RV%e-?d90rPlV+(rV zE_?~O?X+V7*$%vSfXM*GeBtLT{s_vIxRkqbFlJr=T7$i#-HB?so@m5dNjX?yIzqH2 z!4yM*27r?V&U7u75@qE%ZuXSCYx4QzyEmNq#?iaJJl)ru7tc$?#v4~$wXtUUu7Oa< z)Q%q8FBUrvtNDi=#UEm(4dPGLD{uJDb(YJ{-o9@2p~mWUM)TLaB`dGKy>IV3S2zuw z?3>AP8*k8U5(_z0Bl+5$>d@iOVQos->nQ&GLDc#BOu|!#wQ0uMAWC75R+i~&L)}06 z7NN^A2moYfVU2Lo!iqAXnVCu62Vsli9Xf4DF^BoKC}EfM1kkp%h0`_?@q~Eo*0hT{ zO^*d?`>lNj=&9(Y)WACOKi(J9-(E-jutyC1|;Ltkt`3>r81Bbzl z#PiALInVb9Q;Ey$JhEk$T9`@hfxv`QIuon{!OSxEF`2oK$!InhZzN$#9(!;w`neeu z#kd9RPtI7Qvc9b~Qdx<@doE9e6M|cAVtg?huaFPyylcXxg9`Oxx`g0b*m=lYvh@4ThGF^1%d4<>(1p-=TbL#!;`5v0)P zXW{h&FC89#-|pMKaM|IN)$1H*ZN~NYT(93jzK!;lu^)oW*%vF5oyQ*bZ~aPIDIellttfyoPepMS}9!97Q>Erly z3p7hM4fqyYAEQPu1^_SHsEHW44^XMlr}#NtF2NMywH-x-S(&(g%NMGv;tt7* z0C>oa5ExC0$3SV(kiJZRB6J4>41Ie0cUPjdXz#Ab#=*A1s*=V-tM9tb zvV7U5j^dtz(N|fpu&kgm=qqG1?4GBa4z|ubd!&APuy5V;-|yY=^s|>AKIW<{ykgDG zKiIML$!*^fzt9jIY*~Hh!1%|pW|DwbQ*sQs3pH4?7UBMc6-ENwX#^V7?INn}Z$vT^ z&P%i-Lm!TP=zI>43)NP@VKt4%pcn(CB%{nG_k==WPu&7qz^#IEacXv*_?=|4_?;WdzW$-c>31A? z*KhAEXx`Y}G-#B*qgYp7d(X(4_Y6;b?4Dku@~{|~nR#huhJ49nSW=B0gHX?9Pz*M} zOHl?}^fm&CY72 zb&u0lvWGo29%8HgUXv(Z{W{H<16Nea&XNAvRjJoV@NU8A6LC(f&& z(2{8MQ6!d8!F~`G#e?yZuouztOn+B_SLhE4e(hy|1J&VB>AHoyegNVM)pB&uYW57W zk`Dh%9-AmGzxDpHrXEs2nbW|&>J+KMy69AEn!C(rHT;G2ShVc4ekRd$Zeq75T0w$j45YZUb!sSeam^_}jZukZ;`E(sPd%8}Td|CT`!sMsFAAQ=gpRqCZ3~aFx_|&v#0B0}T zXuWm0_-n|)65$sKJ2ZZy9j+%-`Q{Ebt>Yh6urM6QW)KM?CQ&splgYS~8)7MEL6&c+ zTF`T%mUIUl$5p92kgx^JdZ6x1auyaAmK2t_;4>sS zOh8j|Dd9}Ysa>nPl?cNx7tIFIlG5utAzF_>>v{}MrScQ%rvT=kSE%OPy*}G(;^%Z! zfBK9uQf)YG?O)N?hkEm+a?C(hrad<8P)U!bJ zkW*7Hmjnc1wil1+7NhZ^g#oO7u^WJ6)yMJ@4SMFH{w?W8bIK(gMl^qoZaNx0(l zx`Lc;giy{o!{98{=%bYQuXKTaU$T=TK$^*{p>qvzRT0Ri}Kax{mH)v z1;K+T_|9p9h`jJn!s>N_D#)u#QzZgs2g4tymB(3a+y~=7tAvy|1$|~3Dx9X;MQJnw zJMe{GcsL+0sgaOrh@7GPf_{gD6Nx`o>we<^BNZ1G`vHk?deAit?rv?;@OIB-CsJhC z!}`QC+ysKC$HXURFk|(W6fbGSKnG&={C>UHRf<1i==^8#VX{a7iH0CLY zRaF9_mqNVbC?_V$*Gjvkn21B&=P;5Ny%FXzlg>`G7lrdMr}4kGCpih(nnNEy#BR!X z)76`CLA{IE3WJH3cVe%#2>+CD6ME1CA4QoPWdg8WcsWiYYJ-4$UX}-<%H$J=JTX~7 zRg*-=Jql$3-90r5D|PPPYoz-?eUNmiHzw*+_rQAOISJ^SF!$#4S83$*v^3Y(q2y8o zMX@U@T##e7EK+I?Bo3L+Ey&HWtL+>)k6ciFi{3d~dcs@aZ!T?IW!M$G<7B{B;5{K) zF86u;O{GoC4Ku;pPuidx5sAz6T{~v7u88gXkb#Y7k$KFRw+a`Tl>E6)YfoCPy*uHC zToa{2)SI#*N((cY1Ad6sCAxOKm~_{6FcNDt^?I>k0QoMUfNKk+QO)9xx?F|Oy>)a7 z$$WyAXR{n?P^Ab{G+TJm=RMQ%Esy`C7&z(kf1~3K_K?3w3K12G9PBH*H{BuHv!vJC zZXSC$c~3C^Pmo`BT^}!imOGX(10Pd>H&1Eh{`4&LKn!J+a`4rHpV0d!pm9JOPxYJA zM9dt0996}bG{e=sT%BC~S@QKdQC?6fXEW+Y*05c0Jp)$C_eKCGQH{_hypSkLo%<3Vncr@MfdQ46sOAFG!!jTA+!JsJg71P}+uE zgBQFDF6_*=@Eg+Synbi;OSGl7H#I~G(?>inurXIA>&qO5IZCQEOQSMLvDEZMS=$?_ z{h)7wv-hYwbZDmfKnjitc?26&X@EQ`7Vb*e z9ni9TUThDD_tf$K|K0ZRcwNq_-0h*{Y>u!_Go^u_Vd@{#I*~V90*gia#CK;2hJw9G zFP8`;e|4K1y5%hi3-DDq)HUoMiGpHyj76qEtAu2wErFn|&PSQ#vl4BPm;ngpjzTUn ze+q&E0J_1BLu!ux5H0b{?D08$<$NO%SkN%1P9N{Z6Ka=!r73;M|AT`@-C!#b%TFjf zCILdu&#u5HAMx5><2rs^`K$6sT9+P``!zlN{7;R4%+j%I={n>C;B3mHxE_G@`H_T& z_E`*YY_SZxNLo_~0_Yp8Oc#$PUejFHwbSgL*;C~y@wfOuAX?KRl8@4T8QDV8QCwIU zFOKVbht!%2sK?PX@$}uLJo?5x*O9J#o%H^Ut~`5h#WF8En()PYy{T=2&Xrp=YLd<@ z-HaGPkw#frMrA`P{|FI^T86)orxo~*gcay!`29o~oCK1XfZGv;3sK>vxk)k_woF1h zRMre=R3qeaj7FV?dyjqxzY`ceF@4gO550o*s_g%IFm zhhKtwbVYKx14?8>vpu44zFF|FKSsk+S0LcQ*ntuHZVTm9~#(vx-_t9OIbl@^6}n@4c=l;OLX~;;NLX8B^VhgkpHdZ=!STEJlHh4Y{izI z>YdwG`AhvP8teD%-&IvNJRI*>+tE>w+}f~vyw_UlXdNi4U9Kg?5@L>rnNFy~FN%^8`8_?TPeT5gzP*76dV7QtppyKG!1`((c z`fKiBY)X_}f&You=1}y?bRBIWSUvuzKkBgNp_~RO_+&1E;>d;Z^Q@1BE|51LO*JS4 z#VgJFc*SygyZX-^!RBQZYfFP2>W@2ut%=BbKYskdKydrV+H0?E>00hAT0a>YSrrT> zs5*lfV4a;q?tJxdK;-={0&JjVpunmgN&o&xRef)uxvkU@E$N=9R?p%CkPfsfl^pU0 zC9%GQ<^cBwyUFK!!%jQR275BiI@KiFP#vZZr{qiDlKB_{#Z zX^{M^t7EwjB=Ad0J^y{MWmzCE>-R@uM1t1R{Ho&aNsz!NL-e0}T0VLHQN!njKH;e+ z3^PMo0Zz9@h`ux-kPujun0FQA0hWfs&xIz)W-4|Kt~o$I8#E3BeA@YE!)#(hC{t1IHB)th*>j4&G&duuYnu3k} zr7MdAZR($!gDpLkt9|(K*WJO1_qSAE(b%>uKl1jb!I8qkUioB=B6-mgV42sh-in?L zHT?Jb#(pqc=Jzv^nofUXbI{w>u`#NC9pke0E%j{V`xdn`lrx$AtlXcvF8o-0RlBb4 z=lfhAZyBTUI2&VXEnOG#h4sSc6ZWCrE=jSE)&?cnmI`F_2q?Ux*yK|{alt-2F4*n6 z;ek?={AmZv&p$Nk%(I}IIOQ)|k=Y~x4FHv58jt5mJePVqV)>cdQHg2&x|PcZ6FuFX z9c>75G}gys<<82=a;Mv0?!vMuo^qZa1^xo;NbX2Aq!s-WN1~qORQmVlm#y6!?22t`E?m~Q`^-r4 z1?s&^>rx20`~|E_k#HGo1LxSv5y|B2tU~U|7C22(*t6BLykxPU#_@DqN z9?BFVUg&i09z`)ElWW?|T&_c-o_w3ttQd`kDZ2%`YMMm!TX_K0u7jH*7YJhflolH)}ujwO_tt@YJL+acQe4u3vG*U}arpMO(?rD+aV5 zlAn1it4Zq_J%32vD&L2*VHxVF=;ANubkJK`iggSK+EQa@)a)!#03 z&R&+R+9m#Y%pk{7`y(b(N;!Vd*&q0w_D5_yvpq0S9)9DW6@)${$gZPC+9Ly~fVh_Z zNjn^zPK!}8i9K=A;VMMGHwmXr?llG1^vt3`M3r17<~k*~9443JM!-)6$1&)f+Rl-t zWk6UDjnc+}oroPt(X#nZGC82yUgD$Zcw6LA_&$T1Lgp8L zH{SG>jQ{oX-byh!lp7a&6#3h-W=OM!YGc@$o-~P!xAM%H(Rt7074UHw*R5VXI4B6K z*REbWx@K_IAiCiVEyL0dB>K8KTAI)Wr4~ZpAFgz{%Kec06eiWO^}X}t!IBPvREiX1 z%?z#O7ZT_T3l3=}l|iN2*nLQ%;<90`B!3Zg5_VloYVd?B^{D)JNVy{2J8p1+pGSmu zCtQ8)Ehr8h^@*aHl8U92$gdTELPjet6uh%&4lV0+9-0&!8a1IbHyPh>uF@H-AHPc* zk3*M_M*`)tfu63m*5;;ir#Flm5jBr6 zcT}(6({fd#cB+oebX_0LoT|r9v~D?87`y&-+u^HfZrS(4Ck}UXwsctTta$Qj@#!5` zRs@n$5hIBUV?^|hAEV<``yIz^_9D!j877?^I?HR&$%llkiHV-JX4o?kFYKAx>L9wD zDOB}|>NCtSR7hgqmN9Lhyo<^f#L7i<%Rm*%bp!n&m&*key(VO#4p?aj+$B*b$6S$K zi(hjr)*Er(94>AhDqBO^`pfObEnHjgP+!!v^%E`K(AL+FSB|VMmX2!n8dfASFuffLx22_lge<2dhbjk1$IA=E2h+d*Ve&|oTYP|&H83IzxRaeYsEWFw$OV=)X||vUTW|{J{rKA&cY27lGrs~&Bg&oajh0ZbDi!qVfH#Jys=yc&@E8;~?drfW+k#$<{50@|Lk^O6&J@!u znL7HuW=A>b@%Yr*Th8jNOf#&v- z{Hl`f$r#%l(2zv|_3t`|;L5!({rlHi1`Dk0y#*2>k62-{tEv_>899Gk?oxgaMdCLJ zJBo*xTOhy_%$7&;tprTgiP?(aJ_NO)Qpystx9rR&*mhe{n8UmiDdq4#8jU7Yjp9Yi z5USiM;e(QJaFrb(pARAynS#D$xu0PtOk|Cr;;;#OWyk{vinG!S7>ZT|o_o<@ zx#S@K_3$Jk5JB4CuAER;ZFT0W5A0GObUCKg5A1a2=R0?zrBIAjuy<{Bx?Ik!2=}=y z;v2JPq#a4W#jTad!j)He+v8ZWRD8J}Cxyq7%%@NJ-0RV|^ZnSb5n(tn7%arxLq#5> zRAcVqn)IahS&%XhNyjGm5Bk(Gtw@-fPkI1QCXfi=Y`U{l3Q62ksvB|GdCgw8o6m?l zMWy$vzv}Y-I`KBVPmP)KTDXjyYsUi`wN*FLH(^did&WneBFsQUit zfGk1>Yr`{$L)Abh@P*1s00q{WT~umBD55EDF96C0ov+{q<-uL*gaG}Dw?Zh%`3S23 z^d@fQBSd2LJY?^I)4hI&`BE9(@Qf-`04}yy?L@CEbr}8F(4UQcn_Y>2|M7B4oVi?! zDIGXxk$-vak$F8L+h<+U?|QQ$cNAlwd4H2*e+lCVy;E|^2Z8;qA->Pu^I3dB)6hUtq9 zk#;}FQll{rH{hH;kQuAxjp<{zBphV-^4;SJS4IeThvUJHXrG3D^)+hQJBxB!O3oOf zzV_+&9(?Gzkvd<+GIvkKuG5Qbs+W%X{POqCg-eg!bmgl3SJaPK&N_;o{98r1=>F2> zNh5f!mqoRB3k6s0ozWsU6({P&@rh8&>c_pAt4ii7={|GErY@w0AwhBVeoRmX=bp|BpmF0vy;~a|TC-J? zGuoM?oy9nOQJ*(jS&)y6V1^3WKqsdBA4MdriOEd>DG*2okO^L}fe^3IOL3KtKuv9` z+^c;?uvM5F)lkkNPEUc>2#{szlpq9SDMcikN^nMUs>xp!703p^@2DD6@5aBa`eFPf zG_GXN?`1FfeMfRJGQHC8W?k(m%tr4tXEQI|ochF1OE^sJuw!ncM1&#nSCaQo6^q=156jAea>J{qW6`WLkC#@Dq zJ*z$=yKPsf2d-gHeHiIm+ds_iVIzv$eg%8qHR{T8`HtC@LmsrV5(~}J&t@z0il}X; zPjQ}mLpGedR{C)s?{?*TUvod`g{jnXdsiN$d*vbMXn}Wmd7lLB}_vUtx#xnMK-$75Yl6_hjKMGkgLXe{RLcuvzbfMsX*E;N}VnwvQjk5I*K$cwtFF+ zEk~`BPQLK=7qYozp3bKAJC(C_JoPNwG(N7<87%@Q^lRfT;DkKF4C~RjB{qY6tnsz0 zR>FK?Vt1Pds!xAk4{CkMD27P@D`r^L3s=udxCm+NDwF{kUyI?$I?f)h%o&aYh%iRQ zlp|(9M-V7hr@_MPGy-H2U`B9t^HS+_N>$=xvYj}v2oQlAX^cUTZp6*N5U0lydtdZ{ zBwow4Z-s{#Uy2GQ3w-m%#&8%M&LsD?EgRQ&bs#DlMs?o;H%grN;$emRmfG6G1T5l5 zQBylGRys%tJzW7~p~`xIzu;%2>jAL@>%Ixl*>G#0MEIm=Ydh6;{~9Q&Sf3zT_@7c$cpm@l-5#bsJu*Tvl7x+n(?2pV|}tMMGI#OYC>* z^4fiub$3l{E&Em7wectV4h8F%H|9UkI%TO0ebiDWT3wFT2Of6a{{%bK*6(|zVbZs? zL#=7)vPrK!l5a1Kl}2pwy0Y12(%znS zE35*tBQ-}r2AhYD=P6G?W(b#GPz$-tUTg$c2DO9((zc^Qw4xlHn4xb&=O%4C4ADfI zf@Wv~&Bpc11{&&$0wKlVrvl4hB~o4~LpX|EKqxj!fDVy? zgliPgiKdd^C><0@p*UZ1vTSh)dM`esXGNk$^th4}y~~|;Dc~7ep@u@P<=dCZh|YXX49M+k3*6AO_BRD@7u%n|M*hVt>Gv)>9_CF8LyHVvJc zt>+LZhrxy_v5SB(=DzmJ>W{h#yM`kxE7B-3y!JR=TSt#2y8Cj_V-$JfuYDGPG7WPm zPjT!LxtXbN?iCtF2_WL^>mSXz{!Q45+W1#;t`pXYUl)GNv^>5Tasa1*Y2?=-4L&7b zF)s%&h8)237@h<8@7g;c2Qd5kSn9iw*Oz^LBVFgb)=JlfPT`q^qb`m@rL84Mwa+7p zLue7wI*!WL$mm1F)?`A|Op?sg@G2h~weWZ?L&Yqnb(YzaE}pXFTR6gMDWlT?URb%E z9Ze00qC342XD~!oZl(cETO+_w=?*mWV3#?p03}48s(<_SJ9fq6&qP}a)lZDO0^1LD z9&bS~w|Q6mp4koKPcFahodMU@TPE+@wz{RZ$#b`(W@79X_!#_d(LH@2R5&Mi+qm{q z_a(pW_T$(V&#T~vn?PH?kPtqX$Zu^DO*RA!^LkKtJEfj?3V9~Ml&3K_YA?zM1$2F* z1WGh>wb?*#lgT~}$Qn=_RjqC6w}qeno5aEWC&!ju zX_CIJ_JGn`4|QEhlm-~>o;Nn7D9sM;zOyV|wybK`JMh=bpcYGyInr(J-1;g^Jka; zcK>fc1CA+Di07(MzoM0WOT!f9qrR{aHR!B1Cv`3XqA2@{j4L{xs246Vn~~X02{x5M%djvD#Kv$+5dHH5w>8D4XdI(c1T--PRAx5hXC0g232t~BW zQ}9)aT|*IF;8}3RYwNvw-n+;~t9;|aPk{?AYf{`^R(7T)N#6Z8{WN$h$oft{jS44_ zr>C;1DFy_4Q)?4EKC#9aP(tx)tX&nj$>}4$B8X@fREVp|+<;1DtV4UtFxnhUP}aW6 z<}+5i7yyXxsFyV4&zxPC4fR3amldeRMSNBR8{Aq^DvAblR|5!IBRe|TV4HFE1kgWh zGvhQ?3lJY16+Br7PEPnciJy*8fyIlChxpHoN05Nj6@Mh`L^7?u<+uZk>nkapv8|aS zYIF7-ME*@`_jr8kK?t84pF1?UGE!rDsBW55YUH-N;|PQSm%~N-?3ds7?DmPe>fW-m zXKs}q0YE|%I6&)=ykY0>g8KRw=q_>6UCME65)H&S^fa8EFuf?Xh&uN`ra-PBKue8M z^m)h%@4heppjufT=-TBejMI)2yQT2vPJwcol0}CtQ=TSA~HY zDk<{&2s5oOYZTyIaxI+#l!z~@VH(KwL)D>z=S zt)TeY^gbLRJE)1%+O@pv*$eubLZ{Q`4q5<7HurbX1~@G7eynwk(3|K&uWC^V2D}~u ztWo)Eo(%+75)62R;`xg6B0w}vCSav~1(m)KFXWOoY4jOtn|I*}iuZ50X{4*J$#}Rl zaXPxO!fk0s+JnWt;}h*2H`Gl|OP0*?vM*kPtdE1#n-Vz>=vB!+AFEwP>%G8yfag`? z%CGraClK-3&sW>WET|W1w!mW~nLCgeqpX1u0g^!VlTi&_Mx%2HRmP}zHuM`*xJRFJ z!7N!&S_}{Z^F4 zacqe^F#El$Pj0Sh>ox)X{uj;1dImCXJEzgw#A+O&G7G@bh&8WdP#4e;1Rg>rpVHC_ zuqpI1gV^o7LM+vKAaFs*Q0x{J(2;5l&C2KJ5QZFY7HRyW)Rlj}IwbWOL3>qGix6s8$JO!++g;}i7fJLx0FjGb}H!xZbP^$t4>TKaZXsR%x zVo^kp0KjOhsrH53nwD$BHqHt}<@yLxS!|klQ;)!z+*aWwppk??7DvGfI1i${GUBu7r|EUbzJaN$Eige{M;APBBLE z$U=ufp(6#d5qah!^4h^7eDmnKj|GSy#C3_tnKl9)OSws+qB+x}`U5am?SAs`wqE3Z z97sSN^4o#8=KGezT$g+wc5&uDQdu29SGOSXjO_1PZhh-sIt0axQB?X$bHDE6*Fo-@ z<|+|8WVc6CMmFC*y7O>g#k6~C-m?A67O=m?9S7DurYw+!t5`Rc=6ZB^Tz ztO-@$@quCXy+zD$^@H_Ww#Dkpg53vK>inTT%Y7YHsIT*zM4o08IianQeiH#T>tQYM z(l2(Fr{vAGG7uf%cF#_W&5SS!)kV{^-R=wCMzD5L{q?&>1*=t=$~*$4VlVWixp!kV zGIw($A`v75B8HxR1VGL)9SBXgpy{9`q2_w{(Dxl>W1UDOSj1=$g_&tqn+DHiY;VF0 z8bAWF&p-yb2sh zFVFE`{~~;Z!l-;FUW^HY&<^ee=Al_QoN(5MD$1#Z1H?u%9B#yh1T-t8=CLMV%3wkq zfYczHiNpp6P?gX95Ko=R4B~J>2OK(8RfHU%A`RAT92Ggkv({h72SDXugtttC&uNr= zuexFf{I}@yGNrBaxLl5LtUh3ZGsz#c&qEDrc)=&d+7b>T z%!Lnv7Hw(^8iU>n`w2-hRmDLUsy3sKWE^pS&;mq3tq{aWP^O4TfTIb2X{5j`!<7`; z@KbdLUKdq!{xWY?jOaW$Dvtq~JpiNr(~07(Th@=s`SxD8F+%9I0GmKW zp#=fu6-7}8QPw$_J>QN%q{wbV(yq;vZ##fuipZ8g+eMS{AcQ5Pr3k08D0UMFDl(g^O@KE7$~n&rcQ#{(A?uhzDMd&J?yh6p!%8|+eJLq$cSyIB_wV4fv! zj&yfduE~2*$rUJi#G?4Ga?!P+rshyj&!g8aWecgVZ@zWK^3$8!>+3s?TI$BS``5+I zmv_`Pw~USTUADa+0L(YD$>i7j_jY&83^ulR$40Exp-{DTMWhGdHnMNEHEl|*DQW>` zul1Sq^#Qt$y&3_Je-xOdaC-zNwyh%EfJ z&KV}USEB40Asu;`coEPt6bIU+(nnX>?fX@yor4<*(6|nI)tB%qD>iu1k=UFJqUh-- z=fN6FlJ|=(bhQQVi?W*XEs(P%!nTAJ#lH*%NK?VfYA1Ffy)XcbRP=23X!>>8G+GiT z7WiB1qA5Uop4W?2^2;r-Bjw~QMkMr6t8?+3>T9r*x>HG6^4p})ny^!T=`*ZX^4 z0gk%=r)-nA_b*Dl=t}+{_9OL3zE(XUpMA?EHWsQo*a`eiywePI=1z{GY{Y94hSE9@ zA31l-M;9GK^$xY?yt3#R!ea62y=eN2JNo938&Ip_LDy3XUPf?SGtc1VbV$8Jivi(x<8ET|SERFv2e)lU0B%LMd| zH_8U%e$=vub|WkM=oFh8Ma@Z3i{zK#0(`1sESwNOs|YqqK+HB zJLD@UvZBW{sdrqDgBi1x;R=|jA%8H5pO}nQF&RX#g2r7q(U^zHtQCD-t$0Tzny~Cl7eul)e^Fn1pJR zBEqC_fSIr_p{yE>-9`wLN}Pcxip@^tjKNd4a4#qyynagpyBkeadF~+EX4}msAufN1x2&(KGd(?eHX9Ko2=m z4j_3G6{|oc3yWosC1o;J;8%m?w#CO2rk-GW7=+`A#!<#e_9J=Xc_Kj40Yx5Q{gGVo z!bd~<#)ew8OA-!*t7(VY)9h@Yvt7~~Nt#%7RsW3N_rrxZik2*q*-q;A_dGsfJq=-@ z;%p**f{RMNJ#`kF!6=0$1R&A^0y_tzNL^sNN41)nbT0F>g2F{UBVph@;upxa&CVS7 zo7w*@?M0R)6T;Aa-V5EQMCeGgX1zb&m|hPky?uCXczXe=p^Be)dzALhyc`sti@e$>Dr<2agx17K29)m_&SmH zYQF?-T*|jGM-6F=X^qlYqgF~Q8--gj#}Bo$0ykD)8j#w(d@pXBbFQN6`IVh@u_?NH znhLx<*7D2XzwLQ-wV!8KjT1WBiUC_sROWxbXZ>?+%^C5~I4^E*QK&D-}N8^CWGHH{wdYeD|XFR`k5TjbGdM%-qy*Au$>L{JCvyLhFoelPmLY zPIUDboG8sT*i}_x1qyP@0S+8o7I4Ck;lwJ7S0peq#>X4S9E>b5u{2zU2ssa;6IHCE6CUj`)uVVnmhKVTj3V{jewy7;<#Bc4YC*zAHm zR=E5aSp2%K_I1pNNzxr}DtL6uobwgPqaXMdIrcJ8US84ja$hV%invjMgcPcPE$0A{ z@ph$LV$cXkytv^W>0id?LuGjrBYa;EONaebEMXhjHcMuj?|<+>-rt*8n{P=cCLB9bg_#B^U32j4R^Nl1JI6VM~U zVXh0WM+68SF`yfG`7~gqs=*1Ut5!`e9Fv-okic^j#*HZ}EE}DbQjl7ZkemQ7Or$x$ z&*1Lr!o~CPt{6A!!&IwG%8|3GP*5fr0NEfK3{_#~qKiPYR7?s_!#S|@*+^ED%_5YZ z-s`g_r1~B3j}P?r>f_h)e|E}TIUkpN5BkLgY#DeKzpADQEz+w?&d!*AcG;4%GaJq= z>YSg}RMOqCz?hzqGRbL~K4f~6c+ds43{z5XW(tnAPaSJY&q$l-vP>I1qv`w4RGkAG zr$2p9ld-Y!?T#mU7CzP0_0+(9+PhVePjB}LC}o;$0duK3qBu7I|GVi>*IYjoB zMwi%uW8OZ!1OMot%G3_Qp%gso&)kgOUajKV-P5Ic5+mFc}|>z#>oNw$f*ZD zf{ks6lMiv^rTpk>@RMnScQGfPx7?gb=e~2PtF?LWs;i4;w-)nL{-Z+1ZTOFnre< zt!M2}U_EPLaE=FCqq$~V6JlLsnmR$kcIfve!#Q*gxd~3__I!u^qRmT~%9(+E_CUKR3yz&9D4Fn=elk3>E`u9M$WHJ$NT(MnU+wVD<1MbcW0{sB@Vt=DhyjAK9H z8E1w(BSd+&?LD{;xmRxU9-k^}wfeK%D|YX3_wZ*pwB7bSUa{@v!CqxAs4?(VQ#}KN z2j&S;z7*9d*uhJE*!JBI-{$&jL#|t&r`7+}{`w%yXrDg*QzQ0vYue- zfOi@^H{$2=JImME3@3Sis^K`udg#jd{W1Uat)78;6I{1@#z;JNvh5rAAwPr8p4FzX zXDeqfawXC-CQH0f9h=T2SYk0&M$J=>9R6#CohZu5mZZK43KlO$B&26M70#Q%u4i zZYy(C@?O*_$f=^eWO4HzlQ$!$w_}=+yeH(l&fl3Imbp#OE#5fHXJy8pkebbI+wn;Y3+^n$@>~+ z9y(mKy{9|Hg3QcZ45fT-Wmyl39YsvnnmmGh;qvqQ2RgbNV$!TxTt6g0v zoC#|`KL6bLw;x#f6!wNQvgv}poRPy6R}EZVDwL?wuum$r)c) z+}Mn^bjXFne=PiGHj59m%>XZFf= zx)W1ZwQA#Veeiw{4)R*Y$LW0) zKF;9$8}V!S13aK}$8vpgb$IaLODn)GvjKZ5E2^Wkpuk&fQ9ck_bBB*>&>jPcd!C+A zW@D&lh-V05idqK(03s6rff=}FklnNfEe(fo5dnx`70nnIzm9Le)ZhQuy!iNekM;Lo zYL9S?{&G1H(DNcR;GPFSyO(8Z80MF)Gq% ziZP)`!yyg zH3UxFKEG`5OoHWl2H(yHE&-*;>j;{CgV}FHvnDQ@8*3@jX&gk+N$;qIm*{rXYc_|4 zxw)C6&CyYjVWu!sM7SL_JO2b~HX(<=i2qfS?M3eEx1j2qYhMwwUm3z?htT#M`UAR1 z5e9_4nwgLh{@4lh2P?AEKGiZ?#W-3~+@=Ea2}9ElH-9((NH8Cnk{KRbk3dFZOkpvy z!xDtO85M+?RG5)oXm(RQ)OKU-+8b?iZ>(8!qcm|z z_qcK0OA@madg+H=nMaBegSQdBrcTN$(TMS3@HbgQq9)s0V?en|)KEti=0VP-9*m56 zGJ`s7#5!aRYS}L3<){y{I7WoyY)s~GaygCQud;bf!;j^eFB!JkFdAi0wkR!mZ_mDW zdltNRaQ`oRUiR4DGh;=8DzLaXYf?!a{WmH~FVYai!j>Yw>^7(Ief4yzn z*SmLoy<^AM#w|~FKGqc)Gb$@)R7;d4EIzE*f5+7x=5rIdc;Q9iy!tS7q+s|wQ4>Sn zbcC=SQ5u0xs36x;FP@*^phudu-DZb)4}gCrHUS)MBFJu!#7G*Oe9Y9c9U`C4lorV| zi}rlkga`M|8apGm+8p=LfwUQ>vMt`ZQFYV2_;PpY%Y@>(8g^PvS98h!J0Fx`|#`xhtOT>w9uVPR@!a`}&?e;*ag_enQzELd=ZSMA%!tA4(4;m@U;`+9zTVeZ@uzwYVz z`25_t=Ran9`uc}+T2GA=rWiy5XY(whec|9w4Gr=0cEoHF(Sg*jQoQ2?#5L)l5NIaP zKDe*!t0%pu!kiEn!43+X4VeJwp@4&8LL-8F^yuc{ZGa;H$Aojr6L>l~+MB^g>$&_x z>GP9ozid7^<$(p8mVd|x-?!n<%^NHGyVf>$HQn4h<<+j8|9-Yz`fSRp3+ac2O~SV) zrQd$Id~)%P+*dpJ=m$^oK_4!!nDDcLd)uX7pV@V@{`=qO6t+M6Z{g{@V*Imn?+;u( zwkp%7dZOab%I2~x842-56t9IM3kQm(3hqml7W24C#9{y)DKv_c3kr=XGJQ;MLF{`S(XKL%doa*YF?5Rp#Nj&6L zlzJAD)Z;AC|1%g{Z-!la$aBb&R(J{+p@Vyj3_i4rCG(<&M{QPotSKDJL&D6_5jqFBH(B#SbMj#O7ORK{0cMVX z-^091SZ)#qcNLG5fw<)3L$$kh)r^f6r8jbx*VQjoT+S*V*TQkBZf+$v7RVsn_w^Qc zLqyoX``uTU_FkRO{$_hf01i9?o|1^{d-N2@^7e9rUaAd%bz~L#7&3@l2{nX02G_u! zGFQ-ne~N7O#u%z@Hr;9gSWoTXLQa(9p=l{l$;=@^)I#1t$0hQK6c06}3^ueOPNOCM zgGrRciV3|G<4U7f(DoZ$V=~KvaE1#I%E^CR220 zV#kX|YPbDquUqr3va0?3BZW!HMa7B9g=4e@udMso>&^Ao53l)Xtj=uC>T+{+bt~I` zrF+-c+e^e%uP$D4t%rZ$i8JS(seLqW)I;)FL*J<1=bMbfG%3QoIC;sp`OoS`q+wihSpx$|o z#T!vpylz5pkSB73F)+{z2W%}+@fq9+i4P7(i`*G9x^DRi+lxm4rnkh$`bPvrxWJju z`;q;xL{u{)Gi7$m4%0`ALqu#f6PX2cekSWKwZg;$U-ugW?v6WF{L9q`|Fqf9^GNJn z#lKxuaVTqGMMNC7`D_1>r7!jF>K0dgdT7kz@c~O)T;2G(rrGa?82L3J zRio-w<}D4HkN_AYf$ma~vlj?|zX>tvEN-S{Iuh`;;ep_4L718?Qfe{E0ujce=(DKO ziQsh_^z=uwZ_q6Lg9=bJ3g}S%7-{CYQDDCDvF7B+WMhcGuNU%R(Qwhko1Cmo{blwn zZ<)~7D2FIAy%=wlcW-ldpuY%1V7{rbf9%*TjWd_~h4&UM>g9Q(&93gZ5A1x# zobkkz=3Ymst5#G`Uzm~AQ$1~AX66FFub*%T1FfJyl9)eYi%OZ7_0C(aNB_x`B=>MbnAzlI z3U{C!U>R%D(G-=WCYtz%MIz~gxFActf_PqK#rhRA+SC z<;8Lovir=Iyp+~46@G@fQbCb1wHDc&nrHokL46W#?R8cGnwi?MtOS<1v4B?qy%3Y>L)4b|C-&QidSJ z6>YN_dqo8T)M~jn71MbK{km-Bkk66toQQmtjI@O8#B6lv33EpXnh&9tf=cR7M&_VH zg_JzSE$YEI1LKl_g(#DUEbIi=t#2%teqj2THB-8tnBQ}GUghed8T)H{Umx3EKW=(V zRPBV8*)LaiWb}6}TG#8h@VP}T_jVc_tM<%Ues=owv&-k~s&w({JU*-E`Gu+tYxbkD zM(M_ivp1V+4^BUH>hX;`pO*C*z~ps~{J%{W7ZY-!;kOD7azi^i#ZdIZzL^O%C?}Y~ zA7FQ#Y%ANxED_2hA)b82@mUF3zTV7NJeUNfK!WB@lW@Sc!5$;w;I;z`ga=(60f?Pl z$(af9Gm7gvlT%X`ZLvaRojn-3PP8yEtbaPHJ8WSC?Rsg$!m#eBr`I!>v>fXKOgabQ zz{|d3{yWZ{L$6c;8)khhnW*pJ%P=d?jY742s+BfV<@e4I^WT-#viFt1|940JvSrhs zk1hWBvSl{_Ajl=*oT>nMD4`Z~CXkAjS2Ku;I>iZDZ%8;g76;h>##JQ@%YjN2%- zkMMe3wW?s-rSrSy-E%j|dAuwC9kZ9L@5&>!3Xf#whTXe{EADDG)Q^G(@AA35GiEN! z7Ty<^eRSa9dyDw*Ip&Y<4XaQ+_$UzXl^VCMOM+;#XL=Ntai_}s4jSL^Ga+q3`q2I1iY(#@CVU*SvM zILNDB>Yjf?`u@;q-s9vc>6`mbN?)DE&jbsB%Mu1qA4dMW5bh3n3lz*z@)m~h?crQc zJl1i2-TD_g+pllfaJ}P5USC~PUvBQo`o>kcs)u@CSiJE1%B9yAEq-BH>#62hC!1&8 zk7xmVkCGq$qS`3lWbY9IpZK6TJZS5)mmmzaBB-pNjLcct*#GzY?)%4!`{yq_ID!9m=U3bMzuLd+tL;0!-nIDI1q+{D*zJv;3A@zpp-S_;VfFe)N5(+n)O% zpE2^eRQsi+`ZES!^ziqi^OE7L#PD}3F?{*FqHNE5*m_=earfbCUc2oYx9}CNGvq#%= z(9s}(=w&BCW7?V4nkr ztvo`0M*NUx>=P1{XW;rRp20r1e+DEn!*M2`0$@4U$d>-d_$4ytm0&Sz;D7tjB{XA% zCOtgDLW2T50z3jBy5w;3NYOPcqI?U{rJMa^|MsVwbz;NK6T0Rnx9oVjHK+c52iasa@FVXc3 zDu;@nLuGM1I%A>fn@-32zaVa@*Ru%bR1^=$Q3Vs+MLG-!p+M#E+w2y@fApj4YB0+f z7E@xpIm+ao?CzfI=HLK}DLgIQ80Ho3p6ZDi5$UPwTLpaG5Qq@m6@5S@=Hy7{a`1wxBlhw>^V>VbI-=FF1G&mSD(H1Y4h{j*WFkk1Y&`2PR^zy()ZUpW?$j8 z#}D(ub939C{eio{XY$jfN2HIRmp(qVkvEWzFdTPw$}kJX2V^0jgo->XWI>d%0T4`| zoC<4Q1{QD(ty$1EgmP2mJ7~tBZI$It3*cM?cmk--kmrn{fe}FwzCIoSo&k<}&cGY= z4oJ;_^=J*|3)|2Yh zq$C{}sy8j(@y*`--)#H7yu$)CHhF2u#XX=)YdkLBP#$G55Ob`HEoW8h_R+e+r?OE<4TIM zHO9)RwUi=YA7RD7mD{a?3%piYtEDm&M{~F-+#DGdA;(!6;~A0@fd=EGGi#|%Le6Fx zjJ1+w$`t7p^$&l0@ZjGcuCIUi?*|Wl`*8hd9^H?v+I@YtIxutUxb#Jf+hfzpvVzS` z(=!+MF3QNPN^|t+e0=SJ7p*L6{!6^-z(HR9(){@^N#ASMzrN%yNje^u5|EgeII47W z){LaQq$G=9|9dO%<=qrMRj<_H?_qii^g{QXF@*wKE`sRw zj`ZP?Jzs8qXwz#wP0#dq-`A18@Y?c$7hSvrkJ+<51aD^{>s3DPevqK?BA@!)Io|t) zM7N+9q|Z+skffirKKRS7)4baWf5+YY>$@BRo{(O=u~qu{Vh|d$Gd0zkU&9BS&P}`Q zD~Gn^tp@>ZsPUaG!Zl2o2iQJ`&_Z$!a`tR_VYL>$*<|*Lj2#jJ8;TVwc)BPEW{ZLi zHAywq4SOc0kCxf&FV57T|L0wIVLp5H+NXkoXOC%HEsax<;`u%MuGXtxp0uwwqWay% zLmBS!&a{IKk$DjXm3wYBjxbndnXjoP8%_Q1A+$3@UjmP|Ujtoqq`t=1gjk;=>)rM` zlkK~p_sQRNJH3z2LtcY?F{bw^=adinO@W6R@uytWxl#Nn@c&W#DKni~M}-Fa`+6E6 z8&Q#OjXwcT8Lg6xr1%rzFg!3U{uE(N2Jmqa$+N(_k&^*#ScISqB6|3=Z=Ss7(73i2 zRv%g)pWHD!C3j0(Mqf(S?!rUT_b+zOd79_$@>zIkRblT3!kVt~@}9Js^r`LTuC;u~ zllTAq*rfH+Uw?U7dac&m}p5ObG(Q|Ma;~d96U;ot!Xc+Xe%I z{Qu=cTQE7<{-Nm3{Ua}le)>$8La@SiQr1@-_B86wU2BOXzidWYd`?0Rg=Hhcj3Lyb z!^_hHHI>d7`$@i9+k9^&%7wzo@JMiUcZj*n%v5*$Vh#^?H>S`pMvSXgytPCk%%v+N zp8T+*?Ss?xcV!Eg7Wt{%r2GG*apB!sTHJV7^`H11fueusfM@oh=7q}>la_TYSRS9S z)Nkw8`*wc0Jvp}UzxoW)IT!Uj>1?OkSy~^oOS)MUo6A@IUENRjz&Qn4u*UqN zxD4LU;-Ci^m>C&NJnx6ID6+#x!g7-MNB{y#BUVTi&O?x%&>K}dddD_~=nT2_TqSTj z1kWnAdltjN|LkFwk`d?mA3cc9F$SGolM|yeVlt5H7#0#76zJ#cBo#^nIQ~v8?9De6seRs5f zaBks(;`Uu*78K3fE6ZqUjMf{xKN4B8>nu*uj)=kOBH&=Qw$6yb!3daPh&ZiI1>HlX zf~cloB~-MzlhEH1Y>;Nk$gLCcIj0j;FgLAtDN8$sF2w*sj$$)$_~J-Ic$5hpHXNyI zFYBg(QfzErj=L#*Kf^X9f+BCcyR0rNE@rLw!0(oOdqr)Gp7-*ulah3?o;=N)?(3Q} zeo|xdOjN`+WtNp?!R5T-optk9OGo7CUFzrcYVFOd&8KE9ew7z@GG6a*^pA@FcR8xr zg;&IB!NHE?g?>T$>$XKU+d;n(@CriADxa&k?2VWen+HcM%x^F<*Jc-)W4fFV;&u2~ z4OG`!XNVKCru<(hw+Pz}QEwwD367adNPmMiLrHMn2K6va*TjCerubzT^c!m*_{+hI z(qF48rN5tlS^D^%BYfCv7kKB&O5XX>;lG@z`R>ovkL}w3_>?J+?b&~Mswx7%tvE5m z9N&0%H-1;N+{5eF9wD3R=X}Z4BWu?lxs^-)Ueb8FF1|Z^$wv>)A{V{LO+>zRGhjpU zyc}qJfldhbTa{hX_$XW07S9`@@eLYN^12}!pUG^BPZ*J2HZp=|OH3gPAOo>HD+{Dw zRn7ap*|O))J7%A%FW%X>_5CHw-rH2ab8O?u7U{1Py3gyGRJtHPqI~_NiuIHB?W$fo zj$g3i(yUpRRxCMO;p9-(-?--PapUe@)3mYF*`s`~Fsg6G@>OG(PnfWL;7t9xqDdR7 zHat?bozayHizev&c@)QslH+)xA?|Mfw{g7WB$fe?mz?M4LrTA9L>!OOcC3LsIg%$A zd?*3VVX{G}u|cGILs3fXBpC`7AsHpB%4O&PYJi0T7NZ?Fs{3jJmIgIWVlWt6G0S^t zP+ecS0-hxRU3b;?DOSh;E+UtK9PCdJ!wKeMxC}1d5)9YViSi!CN|14 zd`kC=)*mpEMk;Z8%c5vo5|auXac2)siqWSVBvG=Mrwm)HDaA%Wd z`i*nXPQY#~$)lae`8SPIb~zRRDC{VecgXe!{Z^cczDm(JEo8fTq8kJ(dk)>k7%S6Y zf9R-WXPsfT2sRdz5u1T~qRl~ZI?DBD=J#N#B4yYq+f22gD(zEbNo90y_)Rpa`K{GG zhgu$8c=;E_ODAT|wg{?gA|FsVdU7c*9-1*Hwz{BU)*SIyhFHgyhh3b{FK@c5_Uhis z&DC1nxW1{5u?F{ON6Y-;jdNV$J*2inZ3Vp(@1EVUU)I~jc=7k}0HqUsh7CD4*&FyD z#@R^^NJ@xKk4eYLLgw4i=j~5c0S!pgcCg<3Q;Sa59$J81_cq2vMiy6$tw=P+H7z2L zG@S07Ry(t*YKCc&U-^bftve06(ai;QtNdoVRiupRoilz^h35?ajaBu_UA(rpOxjRB zeeLAxr7J5^nkdB=cv%86dtZGoc7N=02wIji8D@YCGijg!K{s271bYZ`eif%jpCUa; zEocRe@H&f=*fdEh7c#TO7)7w=&Z~dY!g7)Mms685&e1o@-NWd~{L3`Y5n2)Kd8oGT zl!QRCpQR!C-~tQ$8`Bd|En^^2X;8mkfAp!De7?UWF?QhULvNL=z0}?P*qY)uA0Bu< zHmSgm&!749k@~ZI{%=0z^UsQvE3a=Y)vON4OO>AC+cLg?y!GnQy1JuRTOa>ELpsbC zrRD{!)|76&zVZU^uz$bw4;gkUf7pD9+%afOTDX)$+)->u8WWA!3HN- z(+9JdC`S&8CxI0o6IkJoOhT)KzACAs#F*f|XU|hTzq#^FUwa{9;CG|4J?hQx9Phf)({rWk_&er$kL*#u8;H2DwtRmh;h5};{{s4e2b8>2 zZnD+$gT7aoVqvcbh~l6#kg{0dB7?qZU8j7?bR~U+8C4Gvvgd~t3fTMtDJm2QWLR;7 zJf?Yq$y-Ai9IRD?`irg?q=5tbrJFBwcfZK|pu+Xe9em9_Z}l#H`&e!5&)!9HC$CyKe?a>F27INwdjA1lePjN@8`91Fy}vj!XVa(u?CrI= z)2HX=l$XnU&PR^Ir%H}OEajk&+zvov*WN-d3HT|+rzX3;6XK^U-&wo%oxZ+bu3h)` z%DR>Z=Cqw{X+1M%&Y2eJkLgj7m=qk9u6nrl#l?$XT!LJNB`+*#IMz^qtg+!(ef_ak zq<@+sl9Ip&W6?W36}U_~+fIw0uNQ!d(pfEr!*NK@6gXI4a9IMiiQZ$N!fDhh*yy4L zeY3UXQ@}W1q$N{3ZGM*3J45)lci@YfpsU-Lg#1AIkahJovGBvJvX>(j%w1LN3^PI zK7uYONhkOSm!cAL{Zfw{X;|&&B(-Xt4-#JDoCl(WSI}!D5K?)g#f$AHB7Eq6CZ4CU zpC;S3tY>T64((zVSleRCIKSiZ-fH{ocohg>0w~&<+!g{#TNUMSg;z#^ptz^R4hBaF z1dk5`!NmTqhHZdBt;Pz2aOf^q(f1?(FxbjqphkTqrhMQ`H8@kant|tzfI$M>0u(S@ z16G95M8J1&Xnq0w^|0R%1!6hMNHHb0g#=~FI~{ognvdURJzD!QT=21*=<*X#Aw#%& z;h5i8!i6jOL3Q`0hb|ZeT&M>wcpGwqCe+8P#sCuHWnWR3n5Ywdg$o$t;+J~gJq!@j zwJs$KC`Qj{p7kK90&Y;PMjeaw$k%CBD7%i{4Oj5Fi1qP|MKI z5_KwHUeNgg#WCkch(mSXgs@1-oUxXPb;Tz=> zMlE<{oM74^o2!`SxjTZjJkEELpdK)OjDBe0%?!VokFZR*V(#DnN5 z(LjWoXN5idO&q_Y&bDXe`29!644%KU$D*eP&aztKLAe=@J#S``Q!U2Dh8Z!kjCLK4 zZKH&PI0{M5s2^bvjFoUKkN1DhOJW#=?AnCh4njO~CSWMrL+Q2lsXr>7;Othz8dM_oxj zPMSI<+M*Jiew@ctPj85fj5f`vr@w{DpT=THUK7k%ZhQi_%HrpRtQ22#%_X8jmoO{r2=bz@0ZBEJHGUhCqJ}Zl{}JQlgpIdfM)FCfI%Sd?L2v}9+;2&Bx5X!g z^WF}TG}{Wx26m4iH>SG)7Rl-sR6%5;8@``AJkH)o7#NiaS!{x}8e@ia#dK71!Ww~d z=$E5bD{&aA3Q%MZ)%LfN?t(Qu63Xwe1`cX;yGFu%Ty}gmzB$&Mh~}z(u^#Dc!U4k) zmQfH1OVmq-gq&eu1(ZA4R)e$_+Z@~Z@Kma zs%M%iCIC5WBUE4RzS>xt9G&X3!>Mrl@=PeHeZRd6_~Wcmc$A=8^z=Ls8`Rc(2h_U} z&Zy(iQz#hG$XxEUB`^zJ;S(H4XN`}I5!D>(veiL;=(mcgi!xhEM1GH!x+`L2F|-^S zBFL#>k0LRM+JmBtu?FoH)oPKPajeiV{iod_ifYV#lW$bB!o$!IatuY8L*hc?@QT4C z_Z?8aE<C2McMvGwRijWt)PMtn7t(HJ;@NrTTVC|$j z0n;Y$t-xerUij$Vr!LE|cTcD}YLjuGN0B3bf6U_N2M1UnMh3*i3OcPLlpu_hSFn&M zlC)!FkWmW|J-T6Yd>gP0q$WK>(v{fGpaTY-gMjbwH9BF{7jL?kN|&QAA>t0 zg_w8c?%?PHkQ%*i$KY;FwQdlAdu{m7kn#YUF3^d$7D>b7i#W_xCdGoIfk1Eq(pSvP)a$ zxAlyVDRZmT7l)-3gj8-Ut6tqUXXX8i-u9Ny?t5QgsR#V@tN}`Pu?}5ec!5Iu%d}U3ara`8bPey(Y-uDhm2w zOq?-=K?rLUX%Y_yC(3zwn133bUZ>Hop;S`z+-C7$Sk$usH-S|yvbkHScUZ4Syi~0x zHz3;Eux4%O1dm44NU`O)+|kIGfU<{t38;JSZVq}bnNN147z*_Y$w0CoNEuI8#?VEw z6e6og83JZGR%EpqgmYUy417qZbE)CycS(QZ6T78x#V(GT&B-^P8SY*B*3Ppee*40O zPaH5gpuF55-76Klm6z)TKZlo(T)3sXX+W~~HYqR{{|T4}!QX^95|kD`Dmc*B2UB~H zDOU>ru!v+qU?fFb6osaJ61kP3&~O}D8U%4cFatC;9c2g_kq@0wuZTt#vU#YLg{P^r zFT1xtYyMN8_W+OX=P&wAt2b3o4j2@chs~8)ttT6{{^iYcWBXmiZ2KO zzPL(x;xh43&KH?}_2^@WcUKU?pFBpJ&(*~e=_oxTK05n_oiG1+o^OO(+4E=PGd|&c`G~$ZK4|&X$H1!`9l9M|;F{XkSdlYz zM(Eh&uJ+Aq9@`h({@()AO$`pfikK=KWp1_1gLF z+Wy}}-~24**V4a5W7MZxHh;qQnhO8zMa)w%K^IP?-hB!2G11{PYu-&IqSYqFt3Xyb zDQLg22Bu-hl*fWb>_LI5)*RrksmYlr2f9B;Es75b3Yj{HM#?uUS-cA36;dJlfGlo5SS6-8V zmWZzUy`HBRRc!Q=jD8#L;Xn25uc##8v-4B;Bc74~?Y@-TWeGAt2aO7Y6E{CEGef1* zdSWJ)j=5WyWym6Qic%>;Il`7V0ZXaxf~cMA!0U8e6AWzG>kxORg<#^fw)HwnXm^`9 zzOVp&!weqgDBo}z|2a}|Vre{Nds~Gb{8Xk?0E=>223>b#j{`qst^^#R-_`x?*VaZ? z`50a1Nv>TkK|WQ{Yp-^6Tw51i>3x*fRnqU*+F#A@EKV+QaT%9f)R8~%s&h$7VOK#x zS7AyC|INX#Ho5Dh>vjl~t8?G<)xm>b;yUkMST5D!_b(5&Kfm11@uuYDxBQy)M^SL} zTmF3eylcz-0Pj%)QR0348pQUqxN%&)Wkwh}u$bV=8l5WWb;!QPNCkm&g!5Aacb{4V zgAxinST{hWM-G%D4cs> zJ~1S2Y_zmTpAZ^1HYR3lTxi0{{_~EiX9rR9S(W4Y{{Hh0>XT7PH`Na3`z?#cyK1Er zt?PtE5AbhjUB@r7@x6{y%{gQ0@PBSgEr9Q(%FX5r3C*)w?0@V>$Fe z4dcZLNDj&vW3-mZ zy7Zu!=(8tZubjPbR6&UT*#p-ed3M5P>7DPslYYIb)YAf$qH4(22d6)uvP$PyE13w;mEv1C} zftP}z$h?$Bh{63WK*|qes1dQA(e8d4r$F-W--4xr9l(-+gN`JGkHRc~CIcq*`Y+r6{9N5u#p5y1|zrlyjN`rKu8o8n``Q);HJUU${|!XWNm@IHRuf}d4H z2D$oG)y&@>bYsSSvyUxWP_=i%jqv@-et@%T#Q6Th{W!CJ!T-^bj#{nApa6zb5j>*l=!p?(?QrJJ>=h-yLe9sB zTH+!oNmXFqLYMTN^hxoa1=$w=kc=X5E!+F%r^RD3CiSoPS7c9yb9KJJ^lL_e$2vOo|(1m&9xnGo^BS3p4s~MkELf*zXfi=@=#S`z6!-B zXIoq&P(=v5;yJX-!NefhSrrHa0Hc&>79av}aH;^)i8U_Fln%$%pYC@1B0aE6+<^ z5p?1?@#>+Mnzg*N-fl=#5gUZ~)b9_R=J|mYyu(Qu?=a_9_$71`+AF$R4%_RkcVQPs zj!456B2wAxRV6%TFEx~*JD@2>LCNkZ-l^Uh!Y}LZY4XokySUUy_m)s-`PN2~bXjTm{`y|uQ5m-;k_?&F*=kx1Nwd@z4RzZO|sO+0C@l=}i_ zjdi+Js18@DZfPz8tC+fn1Ax&&v%JU`+oXc8>&{DBRAd1J25?+>Xh39OB)-kx+hjEB z(NH0ka#iHfx7Ln`%&znH@j?wHluvMta3lrfMohz0#=U?1qW8v9|9to4BLCbqd)FLw zi+6HKE*G7hMZt^Lly5lQwP&naK(xzA!uZv*Ui=-hWggfHDK{3(lh=t6Xa{9QcnTU(sYR3(&&keH z(Ihk|>g$Zc++;*xzTj8c_hEBpp%Gusf^|rNOW?iP*rf z%{!b8NCmb<4haN|AQ%PPBGD}VPDB$}%4p`V1E+;r5YCNe%=Wwu}-XYASKY+d+Xc9R~TykX!7*U~!L$iHVPi_fSA#jYR%5 zP_QWE9r%X=iktTfmwpl?2L8q3k3(^2uEh`+8{vtD)bSYT^}o(Em5p1O=1<$Wh*;xK zzyU3bGXA8kOFb;Y$aE3>h0F|^g_g?QZ$TqGWDzHz39CH`>46P6V-Xdl3ppi;g{<_! zT1yslx2GSr7->y+UQP5vPsKtt9*A^x8d#^~{3rwZ82vbg7o|mHLf+W5e6@?C5&49J zUujvJMN;&KOCGgm7bH)!&f9DAz!&p817piFQrxREOXG_kI>E1(MbLGx{M(425GPeh ztgqkR+{)yI?e=1b=a8L;^fQXhNuqRmlWGy3u z0qk592j@{RFgB^d&CzO_mZ6cI2xD}fZ9NzkThv0WrZIlFAU`iBEj1-MK03mPqb!>{P&nXDRw>?R0?`mxgY04m#S#}$c(?txS9>R|*t}v=l-Ko)o`Tk0 zh4YroE9Ch%>RvoMk4fl3!Cj7Stq|NzCtv$e5TiR@yr=WtWsTL-Gowcd69dPMpS*T` zX?t~b*OB?xKXlI9wT>kB_XmcFboGp)ifmdcN&-QU8JUsk z%ZTLwv5z7B#?@wFt>^J*5DA3!X=PasMTp$M@f}hhYV8@%8Fg8r711eCxg;Q^{}_8 zV>UCGfO`fiuc7o&l+!Rf1clSVEP+(9G;e`?ihdOq4*iNe7Mji#A4;E{JR$w=gB7d= z^$GNpURm4u{J`$r16R8`o*&q=M|!?fdh3nn4TYkcYngPQjP*R_`yP1i_>!$&iGmOr z=eS_8@cMnyXCJLt@ev<#-$_3BSNI9f-N$p+IyZy@A(c&)Yh+LE5dRSvs<=p|R@Kx!iV(@2GSjZFUmsi`1MC=rt#n;=$H5 z6(gZ4PqPc)A5xS!rnx!l^rA{NnshhwE!SZB&a}amB01`VVq${P<~SxdCKoRGu%PJR zXz09w(Bz_=5cnN3J9-ckWmwyEIW6GJlm9J}=k5LaV}>E3te)p9n5eE0ik7%}I7koj z`yCCgbNPohNE`Tlt4AQCP45ueWR!%uF>siTJQaKeGe&i~)YNzm{P)0CzZ%ki!8ehM zP|uk_Rs9x2AUYTL!mmaiMGrvJ$~dDaf~&!dLZp1b8zdmxNd*JUDtVA0)&O8NQP|fE zG=`z0M!L(0d*Cr*3tNyHZ}BmkIZhs^fUl1T^L6)1GdhB#5$?5WzPDqOAaXS#BIW$0 zlh1%u6I{R%C2kg(GM)S-a74##-=CIBMNdJPJ%lue0_@034X?Mf5PcTluL7pShH^A_ z^dWS&1spWI9$pHqhA<0WZx^M=^Xu!0c_qP~jIJ)VlCkX}zgnQld zAW&R*-c>Bl7gVlVMUKqi^qd=~cU!}`>(n}Th!eDP5M0DHo*#c_jEdA;3+rP_BB89Yz%K6 z^6Xkmdf}K@#u=I#>SokbmrfX4GN!$-JtrHfHBb){6JiQt3+(wtn6J<9LkqRWMiq5x zD8GQ@qMS%1MH;<~U?||v)Ckx}#!D*ozhLwd$vo2(jiuUK=5B)jiIpmiZuZ2VZvIJL z{m-=@ZHlPTdU`C++H>yEd5=s_^>}gAoQ5TLu+h?dOhHtPW^&@hFR%IH6{7$;0| zT26<>85|ftP0h+|4h(zrinlkxZRj=R)**Q2nU zf=KUvsTSY^U8W=EIu9|sAm|Id78f0_5u$^^&eRlsv>90%1jvypCJ5lvN)AukBr*ro zaG9~R66)|Th$y#2HVQnSY&qutzMCZC$z+xftZpGm*}?{}w5dz^zcOZSn z5v!aD$_uJVP6}3OH39xUI&^t*=LOCUsGuq>f<&1pgCfPI(xbQtB}M=VDffuM0ezaR z{x4Q6X3Y|(LKLeVF#IN@95BwMH0`q9QfqOon?AK_TuDKGLPR8NbAM~_l01o4dx&HN zg1TDx8#BSYWbqJ;-ZSzw0vQt^a_*-O0Ty}87_|pLddXEu>OaqXpnKf%h53m!mgJd1 z0X~`K4I885?g>s(KEAQltE*~v%#DbjlM~vvcjMF*NtJWv zx5vkqEq=LtV=e#Mj*nKxWqU~Pj$Tsc?>R-6m%&fe#(G!BWN3wZrf#olUO1|1Txml= zbwiALls+RPVti#{ct_5pdp1-wZpbWIHuupFfA_DDt&cu&Pg8BxK%RR(t5-v3XvmHE zivH1Taq&h(8S_b0E^r1|Clk=@30AR318rEZ2d1dmd?%JbIvfu#~_o!3&S)V!OUz?^JuDlL!vcHes$s<9MukUFB&qAbJ3Ci;NLvo{!; zAvk82T)|Kd2^Opuh7zifgTX9%Lq=)tl&z;KGizeKyjKi-vBKLccG{?l`?pS+vi1Io zQBx9dRS3Y<#3`ATr?yV{udsrKyqv=5=)#;88wa-FpWMd6u>KV}g~G!N9$#1D=T@go zjK=`^Iyb+Pb&oGt{?saquX~+Suq=dwo$B0uEvuedUc6^zX@LJgvVY(w4+wAi2b8Yd zQ~c~Fz!dn^DG#MY{8rF#Qy8B>>{$2(;&9$v1Y$p)2G|b1KJe!OsZ?;IAZZ4ilFG7z zA&_~FTFp@3LSz#SO{@v9K=G#r*|+(;_Hk28M5LQq6A+655D+2^xJr2_W_tSgh%)X$ zjF5^b{6c!REYQ>zzhcL0kD0n+0xcgH68$?DMc0n=D-_g;8o~Dyx6p@|A9}=5fAe2n za0N#3Z>Su$EXeOJb_CwhKARAa^Mel~mYZN1>xrE)PbDm9l^@0up}8Qu7+NrN7(_%g zO^^Ysbr9c#t+L!rB;CMg5gQfjVTv_Hx#?(@t>VOy$tFFGGeU=D1f`@=C>vlhu0(QM zX7I~8QM~H0W_}d!JL;^(&oE26E zDlI9|*+CCym`EwA=#YZ4IrvKLQ}PuiPA)}cuw`a&TwYE_VP+wonjU5L^T9Yo@(q&Z zPv^wguzX%PFnPoHBrQ^_oF9cK73&ubB{K&PQmDrECqG-0_w$A!Kv;OpYUKBu44hT%wxHGRs7Fq?Cq+06s+8UdV+Y`jDqN)2s_6eGqg(Md46_xK6Gt zD3FIYBrrNC+Q-|K(j)@-0ERoT0*7G?%)3k$H5|H4)wb(z48e``ocmMqW(>@>!-&|Y zUf(!~2H*dte?#EOvh>7CA)St0QhD;!c< zta*{D8+EmP4^NM79kt}4`uc~K_CC~LEWAH8Z{Vev>Y|cqQ4K96)1swwiIv?~mkVEb z2eh1PZ$CGC_PO@&-#&WS=VX~Wyu81%vcG)kBNdxypNzk07UAroVHuR5k7}0(G`R#% zL8mDyj2G29R4a1Ys0gSN#0B6yMz$Fni3f93`H}L}IpiiFD*>Jf($nOh(K}4lsw&GG zMUL!({81%YC2(eE#%AKnqRnO#T-i|C7;~`go-u`6mf{ROZMB>}L$D#C$@s&N(^o3T zsWzuQd>CgC*2(21N~O*Srde-ar}HW?P=zrl7~*fu!KRuW>f0#oP#>Yb(6CMkck+c4 z@e$-v(ae_~i=zRdbrG4cthoes?m-t8rCc&Z_37Rk``Agp~OWF5QnWUHZbJo)?zg|D}KNWJe)gH1Mvo z?{926)rzSKji=|ZeV0qInl0+}TnKVk+AJQ>Lj_|f1P3G~%51Pv#aKc&SsT$PY*66_ zcnl+-ph5DH<;{@Ksd;FWccjUVfMg~iw;x9l@DHKxM90vSGpIXu#|FhFgCBCln!z&$KM^n6 z5227|Zhv#3Je;EA#r~yBN6m8bI6CF=!~C~)11Y41h8@4^m*-Th{$RIP(|i}zM2*ie zyi>U}jqn@xC?h!>+S5w1@S>vd^X$KPE*rc!EU?iNZtq8bg+L1`@^a{Ke%|GB<*i_^j}Bb;*(GJ z4_w%}V`7Jk_rn7p#R*;UQtbY#4KtrVbl_Tj<8^7rQ^$Ot;R;!+RY+!F!&ePIAYkKf+zd;Idlt7=ud+e0SR{o(L0)o}hL$ zgN#@jj9ICQe*oz;DB4tmoMF(y)`boRpB_Ppcq;V7Hne-U#0)M&zCEfnAYZJ@Z7hP_ zS6OTEOG`yx;_TEhX=CDJBh6Nb$>CUY`Ur?IeVAOR>U0wo1Vx71)Gm$sCqPixu7EIH zc;);PMcRGZhT`hN0qHYo#!rT(OZ7i<>&wEOoC5rd#x)eyUvIr}3(x|#mC{Stkc~gqYwgbBIYA@Xrn^n3j=P9zJ#Ez|nxR2`M?5O(-f`^@Q~P zw7~rETgURY;Jpv^pW|_j+d} zK92t&Of+eeW?R=5@y^q z%DEI9>4nP!EU}zMiEd5NX{K?S$j!!&i;8fsreLTGma$VfAAN;3M6ayYcz%Z05+9^!cUd!;u;C-`2J z>kdHzKBxDQ+J;;&7bn}U8`LSv`bY-P-&YRG+6bRS4HQHrPQnC%$P-DB~I zk3ijJBve-v?qdR|poUSTv4#!OhW5r-$jU6q*t6DIQf>=)1frBUI-8yJQ*T26a_}|` zFo}&ec!P7edw3gZg6FW}b6I5*22WDt$L&X_oP1MsC}OfF{Swz|@Y^p`)-3476pL z0>d)2OiNqZ>DPs!Q%c*JDQ%~volZX>vGP6VzOObfA(`p__xpDSNTmC|y7%03&pqpN z)V&L}g2N)bHGARn>o;c^Kqj0+JMjVN6y$D+Sc2$Cr^{D_OlDFZRvCsWf(Dr`q(4!^ z&~@lxf(VR)iJ|0Lv4pHE7|(LEGW@>0T(}|4po%xcz@TPik!l4laa^o2QYdIMvh)$t z6dHW%TM8APdJ1&NUB00bgb?+#_XM9}De+*=jot&5JiFCtp zE5L&1-8pu7>M&@abwPiXmq1fKj_V`V#t^GP-$kH$WM!I7X@CKjI4^cr0n3)rjbu{G z3m`uqR3J8ge#iBzaiym>&7n&VcVTvOx5kybJ*!{||M3IY@c$RLdRAk3n);y0(i81< ztyr8fqRG$qR$|&r?lGDp$9p|K5K!yMi64U3a$tWCL>A}fSYUe=*x-crC%BD*mhe!G z>kv1P&n501SgI_Ehb|tn$CV4x?RLY&Mx~scxJggYZ}Ck~v=tnyxO`C|+{7{xbXcSlG48?^U=KK8&U&y1*|_!QN9KaW zVh)dbe7VMe15}Bi20STl2qgUCJb-<0Sdi`~JJFc`d@Bq`m#e~6VYd1UX4N4r!ChjO zY0EN4G-_XMHP8)|G#?(O4f)a#(F?+LW`lZew}l?NcK>th$KSuDp+!~exao`6jm^DR z_@%QZe{_F`qu{RQhrOHD*`49pK_PmrZ%O~H%lF>crV-~KJh*Pt?1{`CgQ;;{%c?%f zlRK|tPKSZ?N)gy$J0uZnqt0f9-0vcmG#m+JJdsNfhuj5rLM)WpHxyAY4WUh(8U#Og z;%R2J%DG9T>1%;my@5s`vmp%(>#X?gUcSwDhCi^ z1^z*yzYrm47kQv_O2P>siCT7;m3>faJf5%zF8gqZ=R&bLMjRxMq0e<*w8h9w8i zt@!-J9hviO;x{ie`o(lpj%e+T8u`0Mqg%G^f8~X(6-#P+LgQKCk_Ct79{TXtdrciL0e=o^J%t7ez~?k3sJupg))qK9~5QB@>KN2t9<<_)_`Wn6|ne47C;FRf)gj z+K@-#6H#?NbPD8j#4GV>EG6lgKs~r93oCq?W(~#OW$DEZHC*k?bd%*Su^ZG5c+{99 zQuE^JPnQflH|P4Bb!O*ddG)Q!iw++y(k%0h~32 z7;woTE&=DkIIaFTeImOUMHvF;IdXJsu+p^E9ZA#Kuc(JcaB93Dv?x6#)GQ z%ZVx!ew&FZAWDKJqUOS{m)%oSyS=1;K&{Uh&GPmJ_Oz+P?FF6r{CufT_?5q`F{gBR zN$;W3lA5%Yn%um+@&$vX+dJ#)Gt(z-S&sdOyGGPw|LxqCNCt|2D1A-jYy?mj+kVz* znik8p6f5L}5gaeNS(UAa^`A@*+WEvVg7ByH**Uw{QjoIi$#)if&6?M7&AOIvO1fs1 zolUx1FK8bm8JL*2i#ryv#Aj{=9t3n~#0_QM4Pt5}8*2Vu2{){@Z zJksb0ODWTg+!<06bXYyhN0v1@dkL<3tF9QOPlRU^)S=f-VFPfRE;?H9}vQG*@J!1Z98)t2xB~@v(62(g0I^yv~-nIm`G3@NtNd zif#2BX_^Bj%L0BVn-?o9B+(HiTTtM#qGby0%Y_pW#}w-83W3i^#u)`1>f^IUt4UD@ z(SWHEEI2M3nmZjQ#CdW3BGXVi1lvtBf{X}FA3SGic)m6Myjs+WfP_I}DPDuyDdja_ z{ckc0x|07lv>i66!Hr9$m?vT@5)Er1by<3xPHy zk7FtR0xX5VT9u_hR$m_zR7g?52&$+d*$XjcHfMP)fg}{;+LYBKODe@;6RsX8Jh8^o z)Sf>{WHFn~FU<2+K5)$zOH+AFb~!{@c1`?MG2JkIx<6a(1W*5Jh&^00;stig^o0CxIH z*5KoOSVN#hSwq}BCWUJ13c~zdGQF1;omgXT>c~e$NR5fy$lmqm)Mqd3NSMj--7sbq z6$m#J!EwpxPxm+I@8muou~_N;R_;1|?D{4YL_Z}ms3c(=^arXdE)Bv!oN|H7aV|s! zCT-W0i+-9RlWuuj^X5!Yq9A_FPdZ)OsNZOI`8oJNrLlN7C z3*SrGhX8kH#GGk1;HQ1CA@hNUDv*LpyO4*o3J(l+L6p&9N1(pKbv8nN7`a~{i^7sx za)KNhgp4|Q46;0vzaC9}1!Ow#aL~IBT^)7gzfn9;c2ry>BwM+FbS+Zwl{}S@l~LhL zLZ;EVg4?NWdGQVRw91Y|ry|Tm$v3uRdDer5x|(2{|KN{rx^NP118f(g133!+!0hVQ zh%Up3Vl~*+SU)W+10xib9vUMx;F$Cgwb6L4$eATls~SL*1SxAq&WBPm?c^y7e`s<) z8VKj+8%LwQwZ`04U)r;Mfq1C0)o{A}z@8`~lL7$X^nxpm|I41p;vV$`qOal_yfisG{Nu;5H%TD-Kxp9A)g{ z@B%-QoD2j~Cc>*IiG3EE4+K%#Qp`vYanij!9ew^iaVKAbROb&SF6DCgg?|ZWi5vSF zP#>ybck;zwPWB74n8-oths-bhD;Y7tlH}IXDltb_B>pLGx!9+$cnM;m1XUgY&BEEx z0OU%8aL^@O$FU@J+C%chPWLD{Y&xfNb5j-406h%W(nxXi4Bx`G9*d@qZ7(X`BLU#w+qLjrUNC(Qz#h)sG&wI7Vnz$ z;~ZR(;DSo@ZdngpmNMW>-x|qw!MkWjt)dS*3V>gp@{-ZDOQagAe6WrW-ZqU|LcKBS zx5Z`XS%XSO7>SGnQ2nXCi6l)RHVU6Wwjz*?f$Rws#gGtY55Y&`swhVz&`^0}MSXEm zZjLzvzI-)T#+T`hl$OQn3J{8k^+J)?3_FnJGiq{GAuIA^WPfI*m#UIQ_oEy&C|G0ifZKkb$WT~aV zkd;0dT{390`Ud&uU;ahMwziS^c9U<>B6I1)nBwQVRSSnZ=j5l^B8$`PT6;tEx26)E$=<-b51turZL3Px zW`-IK?&!^fL%r)gzyuV9(%IhGt^7-7L8`380*osDjdW2}**O%MZOZJZ$*FXg^Co^y zXm(9K&wl`pB73YI{nD!#&Q2}F0paZ2kcy!*&6tN{V}uv^+sI~%k10fKQGUcZtQh>yE)L9jVQ$5KA;m2OCUu6{!qkGML0&Lgg_ z{Jxkx$BFk+-m#d*Vw?3uC)Bl2-|o**ez{hKWEOi7M@9z$8YfE9M;3s2pb;=%p)!^j4p<7ClAjTTQ8tn_~GKS)K`?kXPgIo@TU5V1alp@nfK)8NP>Ctke7k%BW+*3O8D0)KfZdgI7& zL-A)I_eR-3MoU*+Euf7XYiHH9m6hhXUD-^ARq~ax3h^T)x$)GV zE-9I$3X;b|ZxcQ3OSfLNeXwRFpDng;y|^v<#F0mj-8%6ObW7sTHaLKZKZ&9@M*wAv zj4t?|QE*Nvy+TDfL{}^aBp7oz!1E|O7AYx=puA=QK`O2ZBqXVj=9qYb z;x)=Rq&N~aTNyQr0{OYFY$S!&@HKJ7ndz7iIt>w--4b$mIaXH=QX*CS@X^Os7FccD z1m|yfQ!*beEZuQ!hu`0Rc$>W@-P!O~ZvBd<&pr1TF{ZsWS~C3LH#bU$re#E##DXxf zckc`Nr}T9-jSbQljy!tZ>4|r6F)%uE=&M_ST+uw_*#oU79np0Y$jj$}n`#jljparM zkkW*bqTSH!;R7MKWsDpFkDqLW)IA1*NFS%0R*s?KK9%e4zpl>r-u25PYabdp z^A8=(_dVXd<{$1~{b2N`-zk_M6#j7h-u~>Og@<>2|LP-cc3boAjRuD?KOzkB!tkS; zx85{+Rp~8(#{9x{Up{{Q_pg?uiC@g#*0yncmNqzNM{~o52+yxHkcB75R$l_0=1Si`iD%Zsf}|HNmIz3SJz>-egNHthL_wX6Sf^`n12aaZ*1KW22#%aUfDf7LJE zm>gW){yJa98*Z4tud8eJEe&%q`k&FS{@7%*7?J|4pgZNQw5ti1xCT{D*@RwE zp6xW^j8-7r2;r%c;E9uBN}!(}QOBW?~Z!#3YQg#=@SKv=U z8pm>&v}2Jiun5Jx;z7s)Vq=&e3sPvPj3_=#4n_JnAH^b3(7}qolSq;hzmw&h z-8v5xz=gnXDBQiTGJmsweO_}Qf6ZthY2^ZwT|Zi|y(zo&N2U3J{`ODbRn{patux5o z9dRU9itbKAE(8=UHYVsxF@3p=T@4^=Nv+v9-Krte<)~#@(yEo}cV*S_$7%V>c+}Pw zfKCO$0(4FT-DNW9FOLjVEqN(Y{3ri(j1guehnPk1IN1Ak*wOc3?}HpNSS&g;J_kmS zD0w|Oh@v4B2yZF0dU)g{a5p!OV~&g{e8L<;O5B|1oOWxLIo+V=f_yOEAvzYRoVu4b z3Kqa|E$sQyGZ3XycT|(-CGL)vjQ{d>=_^zA^!$G{oZN1aSsKT#{o*c_lilBK-0vgl zaII+3YDD0xc}`Eh#Q==pD)=*~Yu6?dPZiXH0k?_XhymPsLM~u=n`@Y_kO3JH;*sn{ zsVArmJVIWS*b6|F%XmIg7k?Ca)hgt>5gOX`?;t-X`(fFi;wPP;xv{pbuBEc9FzC&9 z=OEH;XZBGVDhHt-q{r+<7)aA8TPR+qzPT)U#Yb5}R4GiA5sK19dV+Ln&5<>=P3MNZ zHtXSuUmvttt+sEpS%xB$bMD00eMe8{7zFzRbs zxiJChLln2oCk58Q8^mS2(Gb^Rb?wN#g{3TLdaw`HOODgW$5D3CXTR?fLzGt`^e0cQ zol7&SjcIBGgHaHxQ;h@rp5sw4oT5b=jfN34ePijHBdnzjHVa=ph%hR7b`Ar<~ z3sm|em+?YiAf>*Kmif1t87TmNyO~A0W>_RMC=7s|iLVwNB@;A5;&Wn&WMNR~wItFO z!T}y$IW)L*arc~#*==KyvBq$9YpA)rOg1`TzM-3nIcC1iV06goTe&La$tAd7HaVaL zUM9|x6k5H##piF`J(5!+SS%yax-|s6w1)rSpZ`2?R-A$o+|B3=y|KuiM89@m8y$d&*dgDOjaK=_h%^HC!)Tcz&n zp38B43+68A?#Hjp>1c0jtqlR~E1af=XV;>5?P7XL%xlLWFk-Y2>=x)ZsBc7A31XSJ z5*bpAC&?fYQ*2@3^&eSWY4*smBh%Z$tHx?g=loR;r!B9z($eDlt#j+O#-8%I8+=bk zf2BAYRNc`Xf+^5n(s$Re$j@jR+ZK=2^AE4MceuE%wJ@)#X<6~LzaE@;rE|qfmCctI zT__Ev*vfhca_YH6zMPvQ7B5si#45xplnIGFsEUuN&jHALtwuycC6=Ma6iMnlwGQ$T z<4foXZ(FO4(lST9g7dg+pz%ZDi{O3i91nAi~BxIC61B9n3$MP9laLlL5 zKgE2i(VI|xU@AAkle*`wxoeuz;VLcdaip7j1iLNMw}afO@c)MmgxlS#=56{33;9)- z^ZSH>9ZS)>g?JW$FZiSX!j%H;WnU!2poIwn7Y;Jg!I`AAkZATnK_fQZj7Qc{@jbJW zanGP>S^}f|GeYpcregC&+gr<1^h2-s3_rh+O~__J_`>yZD-61x#w1gSL4 zX7&XWImlj3)q0zC1OL=!v!TG!x2DOFS+#w$ztlXm&}FpGS>Lv;;}C{u5Xl&8`xwt;vob&U?b3&bF)9hJ8 zorU!^?cjg>PvnY`oBar48RP?u@V)b&DFa2a+`LW-@Vu5n8s(M@MAD@mSMtYj3@RT( zyCEGS#Hn>!omxAd2C+x&F{pp7MDXn?@eWwZ3hDtG^Ff`VUyDBqFa%C>{bjs>X+p%* zjj>;aj72AIaXjf2D7Q+Q12McDa$!>-X>sKbk3!(E>;Y0Le08-|5LNJ9M(SE+V7s!? z{9Kzg=D{?^wCdC-3{C|zPN$8-AQ@Z{-byHDr zK=cik9v;2_{Udr!0_Nxn=$7SE8`%^_%Tk zS@z9QajQKmqcMNy!DfGL*u8P3aGSlu<_u+@I@@|SBTYRZner+QUAKSO?!EWUx1Ju< ziaBZLUrdAG24*Av%>P6X<{$dp-4RcYTKHFJNM*AQ4nA~~Rctv=2+(zU-v#6LHRJ`1}T_;_576=ggZ zswxXtgo7xNgSqI;%s_feDPKzA4MtK>3VbXF!3o^el+}RhvQv=NbNsxC=WV>zxk1r)zWjjn0h5kvVtm=>^p$ zEP5Ne1cfXE9~N5H)?ahA;K3EQZy1bAv_p*PKg7Q%w0mDCP>Yrq1dHK9pm0BC-vI@uSt#L z3|gyxEPle(0}pFH_tQvv7O#UN1Ch7@lHW+_pd((63l$x~w1RGv-Ut+PT%&|hPWU%! zU=eWba+#|9XGZU8+N2qtf&@TI7NRk8YXX1J& zgEH59JaVmzvL)K$zKG2Jf!gB>UwulLxXsmdWZmE$odaJTJMiY!Dnoi^*Q$nmEjtS? zVQET6}%;LpjzIcpIN8=z53fHc)LAP)*5PrBhJf2_yH zivJ=EaSZ<5&Eb~U>VU1l$5YnLU3sBj+>svElOt7dtZmw_7e zF0H#-t6F)kXCuW(V>n&(F2{cUW*qD{K z8>8PfWbnT;+64dh^Z&BhoP!H**MVk#O?}RoeBqfh(1+bskyZ-*df0ULJ+1G#fe$q$L zqvPTqgulR+DdP4=^Z-Ie+BLKSMVuh}NG?K!GXe4CBalOoHE@Me76@qxB(ouaXBvgW zz)+)TEHXw|{$`##+iAkq)*jt*%MUX1jeO=aS$V@P^H(*-+$(1k9P6qBqjx__Yi^4c@qYz=NDe8F)^Xuwn zE$Lc*GU|6|9vMrsHTr8p@tF_kGs_E#s|s>3^NpZ=6=c|lzy$eQ(H?ngm%oDFH*d-6 zGG(?N?-Vq!~f`x>-v*mtPM|P|c%T(yYyJFZl91!u=yw?U_NHxy)Zw1==^Q z*ue5?x{#OtA;ZrgJ&15Mewz4PLpeP3c~Jfwy5rCIhAFu`^o~zegsCCFGgS#gbmTmU@+62yiIeNvDYcA=$Ql}-)Pj{Q|KA6hH@%z;oI18EvX3>&#Ga9h8@?U>OgS>>+6cj``F0MiF6cW zeSkkjNb!=J$h=%^6WMm69Emp3;Q#HV0H_DbV7tU7Z&hPB3|Sq@~d1GSs+Bdh_z;RZ#l}T;Se|dxQ_A?YLXY(b>BR zo>G;QV$3~AQkw@Nf>P{c!({=I@QITxGlj6Qn0O(J0LttZ4MLaDM;`C9eFiy``ityOpUk=$uP{ zD3jEg>2(2R!8|3>rS77#kd&29{FUHcQui&U-NaW~0rlof{9O#^1-0E+TQ=54d(|0f zFGrjRHMnXj%7C+)a8h79pyqBI=bVB{D7zCiD5}+9foe4+2S~`zVjE^QGng<0GtUZBvy+)fTKt~ zlsP~?W!x_)iBcaSP&BpX2<7-lLzFDZBO49TGL=d*q?}fbst=Cin&Lu#fd`JPbQ4~0 zEb!^o4rKZ;+tY4^I*)cjm%nzP))8>Z!Bqj^rabnUBoOs7gvyN`d-$_S!E=Qdzs>H~ zMs9W_(g;sgF4S|O2bF!@@l%LUH=H3 zUGY;G6FvyQXyk|tJEn4+FrnNEZ!2qCcXZ3m|MKMGGs~~-8T;-Wb4b&r58K1H)>PMS z>R-AkemaF8vgOq+Xs)_#&#ezuuZ#?@*uO1%B)izVCU4$gvA1I3ijLkL4Rt$~9=+cK zzyqH13x7hb`%dzEL?X!FRuiz70OZOmSgWt3;1X+9F4Ux@l8(V5qKTFAKZ|R+ohxqH z-gjhiRmX{PgI6k77o_`Ed1rUjg*)O}vGkAZ9`%$LPn?;qT?Vg-1+g?k`kLQa!Zh1gs@`k%y-U}|w#fqn$l9u-6y zEjJ!Jrc5HSY%XAzDTjC~OP*#>wV^4`;RrA&8|9ee`N>Zqv6XJE%JKqlUare&O4H%) z<68I@y0uuDNJ1lF>@h_ti3{{#uB7N|!cF~$h8J}Ag%mZcdB>tuS-18&66*QZ%uq{P zqy6FRn&r`wxx0?+nya)XYMj$l*sx=o-b8A{{E<~H*J|pkYvvQr%)wp$d(g6!Bp)yp z#bO+Pl6>%|3o1xHc*)rfB`OkKlHzrWKyxYd`{0xU%oTZH6^V`oQgBUnE@?Eru;P_c z!;LMo3%mm&Tv2a+_1i-?&YjciS+uya$7b82%|E{U%P;NS(?emh=9T$*WBR;@oF~up z?^{_^SQ9R7nwM)n{b!Ts#C01Fb(A%AG`9@o&jiPC9s&3Y8c64=H&;A;cSoiPuJ2pJDs)hDJ1^xKvsA&yyZd5us>=eXcg9%?k$z}{H~0@?3F|-)AvX9*c@7onITD zcTiL$C}j#T^4l%4f+HI+EHKNfqn|Po>&{3opBrAnsINOMef?ZnUQ6AhXgb53cDcRxe5a*|Ho@63uYB(L(>dOIzN@{pz3wz`t0^yCO1?*65oxhTGuONJrILE{AjuK!j-Ti z7Lzzj<;a16&4z?n6%UXF>@>0^Rdjj;?t0|=A$q2gHbR9!X5TU@uBNb$C@iX|C@HC^ zu0TdjNqI?mSt*?|Emb?kw)wM{ECM{%kAFV%v<*!>D6E-y@JbHdg%3Au_$AtBK)3x+ zksB4L^;#RTVx_zQlzhI@fxsW~m7(7$oZtWlfVWvR5KK?9P!=Mth=Yt%n12geRdIED8#H3?EvW6#>N*0`q5Y-j3*pb?Louj6< zfcNyl;XUnJS~}YN-F|85gS%EQjvi}ku9#I@)zBrA#Mw13|IuGZFGMFkxIZV~(Rj3~ z$T?8dy=?cQmh~-mzCh^W3vxUg9v*J_4E0w8DQm8W{m?v{Pt<{=&iu zbk&62j$Y?fOhZ{kbXsIu!H{RA%tS1S>uYOkYHDk5Yo8TqYHez5X--gw6~q0{slz9; z%N0?HFHUBlE2ozDxXeua#T%l%;tAF75qUcNd|N9{3td2j^5TLxXQl%9CLZsHF#!Qe zu{_Eca8e)+Bx3~zu^LQyz|`-k;*x!baFH1~|7H2PZWr$1+5BuZBrF_#V4ut|VX0FP z84Sd347CqZhcymh_XY7630vIFGK@9-rV`n_GDq0ry>^rfqXU`jZC%!D0 zR-KqT@9JY$&s%zCaMt*ezGFRm9~mt#a(>meGunIL(7@05jz52%_aE@hui4+ix#rMTu zAV0g5+Zxg5!n#D=P~5lfoorynJ+=ZnfXuZXoN*A&xa?Xdkp<7X&Sxq0S$t|8I`?4d zc04k9e;d%#q%qUx};lIA3OK*Qwz_my5`3x_rBP?H&5KX z>d>us-1zmk`J)?OJLD_kxBSM}Fur^JiA4n~ZX7%KwRM%Y=<9EqXPp`wzNWV8?-V^2 zb|>kvm85&2YmX97n7ob|tZAReo#V7KYZ@hK(^}Jsm7t*}WqL5(26eTx(BzAyHR~*FZS_0Rx4TuwT5a%*D!xB@MGh3plpox zdH7Z_hJ3#?hWTe-7xTGH!kT4cpB4BQ#e5Wx#@|bIq4IklpXt4t#F#(LG$!e{*_g}v zZSqyK--fUE^{HzDUh{kC8zQesgZfe6V9>njbLNX?Yn4Bje{YJPyBFBa$?sSke@7_g z^IN5C(2e@}mC)zFbZ#e0sEWt+ibgh&3&?blMM@paA4#i`#-Y?}CE*B+ew9{G9YNer zO&JC#YEjq=^*-3JwUAQVs7$_QrU%uqsnke!R33x-GVvIaT@Y4nK+9N2BZnQ~LK9~S zSo8oW2vZy|EGyM;M*EY`9nO|)LsF?)%GrZaDKjmGyiptTO^Y|-ARhQe`B9f()8}vK zK6Y;TL)(_zHi*Nx<)zts^Y|=XMt{G29^Z;~#@*EXGhfHy3^{Q)({fZ9dE>k1tY01~ zTyZ@e&yPLE~>jBMNidpb6uY8kl5)LO4&Rals4+7PKYTCg{V; zPCrYCv653=z)}{l1mkYMD<{(&v2xvo?}{~-Pof6382aQq|Hmt)4b?#wGnrjUv>~Zo zflH1gcZ~o3rnFJSHZ@#Jri2i6;N~%8p?^5=%%V!ti(1z%PSIjHuJ<4QN71se= zige0VJ6bW?ofPBSE$8XE(TS(q9WM%mt~=3*BRGAL0Iz89$6?c4S}y6 zZk0vSjz8YE;>6Cqw{PHWWrHi$hT*;0?~$dFORWu#@9tQ)BCJRyrZ#ATF`0B0bdL@# zJb2@wK&fThHOmuzoIc#UKL;OH16Nm1ew<3?$C0@ZCEC-va~Siv##e8b3=ZCR)KvbKXXBSfw}1XX!>p6*M|V{C z-sIcdMRV@FYUjgOmAPBz54UbVRF1AxX}^iykecGg{QaqWjWRdLZ8i?iT?O zDmcP{js-tq8mgVNx{Vx=We|f!5hO@Bf_qdU5#`o064gz4G)AMTuOO<$l5!(HEE|cS z;l^-dLw%?&1Tv*kF_w}`7vN=6kw=ZBAO8q9Jo(QfLyzn@eDk5LtA-`tCVlIJoA$lD zb?M0kp{q8|x@*tCT1lTJeh{>sD&@wy8)h}Wf1wK{($s{TCsfHNd?kBFnPQU7L^Nq7hAKg4}cT?kHtIO`px7$2RE23}l znvzg<^f}%btjg3zpM&CsPR^Y41bQnqazBrld) zP;wf0t2AI^jX>bU48MsgROH4=5U3~zmJsVwtAKe3gg}j`LR2XBfGDCOI^`*{Trc}a zE^j2&!N9CFge?JIQH|HCR@65hoG)VwAeMYl}tu6LkD?hxq|dW zQ;+L|iG{WY`G4Yv+7`Q`8|NRM-*D|%_Clv^Hm_KFPmOS%Gc=B}Ie+RPpG-c`%_ z<;Ck)HVHXhhlX9hob{z=i)x)cS2fMQ_KuC&ZMlJ5gWmmPy_8p!SJvV2ENrmWViRaD z!pJlF9`6KBbA6;1jzo0Nfi4cNM}!m}G3FM~iSa{uCEzu{tTTB|7 zLlK-;ud4`OyD;LXIRL5)E``L;!rveWHzHp|{!Qa0e)IL54Yl1JOO`j$Z`QbLjbe#0 zCxYKB3Ld?)C3t=m{%9P)oVa;o4&Hc*nLz!bOeP&p9^mz72 zRcG6xCG~aBJ{ziXRT-rR(z2Vn_-lg&2XAhOzSeogMX@nqHs-$A8184>U#7gr6V(f^N}8PJHolx7~l`Ng-4h@7)D-E^5?zcee&nX8%q9sF?|kM$mRndwF6z3 zjzmnT`BUq$t1?%~p^YN6O#;HIW*jXTG_1gTEegCb2pyqBlvBxUCg&s?6Z6b{fGtL^4;e^E( z5f;uxFRa`bEKz^4t|@C}bMi_`7g~}MWj@wrrL4$dVCN3;o7Yr-<$T=sh~A1hr(wRa zhLHs{V_t(QE=nhWOvTJKDiKNsJxa|LZ=^_qKen(-j)Poet1Ga1&NDU#6G9{>lqQ_GrOH+076!WSRHY?JlX=Y~P2&8Ay%2%o(iUk(UNw5> zY7hlU>;@Kr3ui2OH#8b%(!fZO*(YJ4Sf=NpuyA(bK4D!((dT<7QJ#Nm~ha!NPNHgc5C5 zgAO*17ae@y7)HtsWB#OO(+GJc&H>eq-?eA#*s)k^=>n^Ko`VZqwO?@WVu$&wFUYN= zy{8W0R393%XY}Wk%5qFzm!MYv9{eMpo za|pwXt-#0ol6w#(tpI`LsVg`hTLGG5CoLgbvh#p6GP^yt23zZnbsH|E#TUVIX^=lh z!Si;+k*kxrMY+c(??;lzk=xj&a!X83hJ~ZGU+4m9Z(TVUpHFm8PN_T}2uOJrjBXal zlcyk0{-?Tx`V#m0Q{D~b^E#HUY;GW3!daau_{`a}=L!!oU1H*)d96m2KeHSn@ix#Q z3tT!B8Gvt5$bmBhUH>EvwiZz_$|#!8qJg+g)Bul>#_^n}rhaJ1nSv$7(m0c&KUS;N zYxNccaJaI(I^@*E&Q^*v=2Oy%#M_u$Sg+hHkjEEJP}@=B8RfC3r^jr5YkaMPI4^MZ6!(!gUI zCF`b%^JW3gt#B?Oxj&5B`}?s+#e{In_9p56cNnArLbin$4C@)z8&fZV%X?4cieE80X)3UC9GxAl&Gq5h znrc8#SPNx>2gr`4+a)}MEK$Y57hviFAJ3j>wCWF;S!ugm_sCx*erM!D)lU=q!?ZK( zk92jkeo3Zl$-cV7rA33$(AtuFd3C@4>g#H+e$bV)r(@B?ZT6s9rSKQLl+q!}{?Oon4ah1V{Tb%2jQb`==PyWCAD-#>Qp z@4L&r)s~9jieq!;EwOa%>|d~PN1?xNPO-^W=qOoK{nz=2x_Q3kP(#D@v;7;I*CuHT zeZ9`Q@||}+_1k@0Mw^?mj9+tYeQDqRx2k)tTQoM;TiLO(Mx`$8WOAWG*drdpT``O6 z;dVr}+Cb;#0wd@<7xYE6!U!#ADt)y8jne87$N^n}hqWp=32 z8SW&uJ-99HyO%BoL9PPKw=np%jawNRs!ApGv~B>aJ@X393tF!H`UYQAx9`)8Z%$?~}Ez;JuYPjw2?xt0! zMKjjthih9`$<-njt=^o>dF`4*m#7&b`IyW2E}iS+eiX5^=LsfFDX&82)X?(1~bwGtQYzX(PA?w*a>msoJX%?pU_~bC+9Z;YST61>e46R*gPTy7UjcG5Q-M0Ez(4UU~PIu9$O42dm&e zV!UhO7IOa-u|#sQXC>G(nEdfQBe=J&=?Sfa{lA+_<4meF6LRaZWk!u^Ben~Bg*tIm zi{eLEvI@3L;QC#WIWxV8ZT{uI3EMQMvwc=;b9H4|X>lRjrT}eI?4;DzhA7D_Y_nbN zHcfYMgs&}XI@exUuti;a>JI`2*-ky=wW!D?3gk$C1x@7QV7p?uH1m ze+8AM5=rm@*eF3z*MXkr4BnU&NdUY85Q|DWG#XihWR5fDND?XTgL4zdH`b^$aGR@D z%uO)EIE2Mong~Fwg4OEh8o?DNNo-Frc410q zr`?ivG zJh!`|OMHcVa1KL%LaW>8_N`ZPkSq9)MICaG+ql~zIgEZ`m`PQjVGuMd1Pv+wll5EB zgN8+@4N(XKZC>Pdv8MF!V{C*mrS#kdDFqQQ2!$zQQJGHt>!%wEI<_{0Hla#Xp8I|I zd6SbyfjtF}1`{Mngp4_4QVAJC{s;y^qZ4%=^B%7ofN7Nu@!t^D}kJ}-! zfAF*Rx?Ow-WM%e5iRx{3zIzen-ko&9^#(fOb3s=-FeA{5T&qc@E#OFHjx!K*g3^LP z4r&}sX67cE3UarUgq$2pa-|D)q9t7nXDmHh7*Z7=tL^HnKH;7P>2 z!t?&g2g&K-L|<8co`fFUgdEIGsHxF~k<60|i!F%oQ6WSDI_iN!@EYhO9l3h+D6!B( zZ$JXLh)^MQE9POFOOkFi=*dD4EXyoK32CUWtF>7CzN|pN>Z8{7$ciWFr<_zk;WPV+ z1^Hmw2KgD-)ygAvRXTa=pYFQp<&oBww34();j*fcTNaGmG|T7J*7Ij4{xwxh3O|YN zIsDy&rfc50X5N3kXlB5gA0itVbD}ofRNl7{8yiEV!Mp%Ula+;b zefBH}2BZsHczEP-PqB1i+)K>fO-yRy|DL&S!QnZE!Hy-Hmh~5Jjy}TwNjh-jaOeJw zMUL6G%~`ZH)NU={f3ui+Yo%S(3P+L8YUfr(OzFsEM$870D*Q`H3vdya0fkbOM!+Pp zeZ;WGSr4!ozom9y5gk~@Y({=w4!TpKO&QXdS#(~lkzIx8y(%2NY2}LOFZlfazLH*R z@fZ5wYxPm!NoYPa$}YXF2PX#EezM&5moe88oSK zlo`%=02%`wNJLtjgv2c%DCq#w1f*P^i4A@PsyN>VoJ^WwroR+TK--T)K;+}pk9ThH=oSuQJ z9FbxTN?kR(;Zp$`8;5T06kE^b_zy$P=XfA5mQEVYp$W;1!?DV= zG>%Jiq#>-4X*L=0^9GOEqNi4KNq#&(Qnmv=;OP*v^S9YLqu=ec^Q-DCq9r=6P8Tx6 zs+QSnBW> zoraj1<8fQ@d}7yu$d~dA2i(d_JQS%>##8zM`;qA$h_e9w7{?R5T)x#xr=5a9p8T9_ zyUmnk%E~m;nFlWzWhy0|atoEQ_|29XSmIYTwI$zIDY+t)=>&OGhD`iN(S^CqTKeynH zG2v)o5+%WpAjNjjk+`vv`vvi%ipys;=<}bUuHQA_JWg(1E$g>UbP{*(6M`!*hLMWh9{Rq=%vnn&u^k87h#B=J}h#o3)6eO>yH zVdS<756BsRl$UEjj%6OaN+ouvG3XnB)?=U}vGk;G#l*LSK{@H$q-X6@zfF49J@wn9 zXUSDQ>CN$HlfTeoy0I?^kB-_CP!40E5h|I)qrEpEdD@2Ve{}rOrmHZ0P?aF_J00l!t0llc7 zOaWZbdI9hrpdb9m0<7R9GDPYx|0yVk%amtD4!RrrUnHjrc;8j6RjrNnr$RfBs|+zCG z?2*nLaBE!A_pG*K6F)j;vqs;|7jyPWrzHDpzVmk^buroIdySt1#^iWo|mozanEt_fn$-`pP_hI^k*G1D$?c##`YO?0g95qvAe$`}J8ohhK zWO~giWG)o`OLbz8bz&%~{psV@J#VtPJ7LSb1G(qqj=bQ=LRK2Pv+0t_ri)p{@w>?i zS0IR~Y10TZU@9@frHr8Th$%y6{&rMFEr6l3;76TaIM8j>&q?8DwhWnFV+-WdMy$49 zr;E;=;`4hPV&?l++ijfrLjVGFfhY!H5P{37+(jZO?cT3C37I{9H&*p=c2DwFO}W#YJ*0h*ud1wwrNe zBtJ0-aL#a*Bb_JqT0vla&%l>3#{)KOEOJ9;+E}wE3`1Qi-{@31KFPG~h!*Xzm_K!i zj`iYu{DlJXXX32ad>1w)*i*cl*c1IO;xwVUOGXpaUH(@pO=wJ1PiHhXM)$aM`;=T< zPxPI{7}g6Eo5-FWOZmK4x?TQ!;=Pp57so!2hEhJil|E0B6CUnM&*{lg0QH;XL=lQV z*q)%)1kM==&KNiic*AN$h}R&UL9wh8j9IKaAV8P?tS2Ic@xky9W6KRZ;-ayq;xCt( zHq(=zmz$Gq&VnE@o4pw1TOD;-}MJBXTy7^!L-R8e$Z(=`8lz%RLz};Sf+TRC2>Yie;vu z?>5eYM4c=^k0Z1umY_}t0SUzD7^l&Uu)YfptgQ%AxX&2Z`9xr)7Z zR6~NBdQy5hZ6!ATBLEJQPJEPF%f=*~d?sA1kF3%?_EC~@wB?_^7jG0RuTS*~ z);Ehk88Nl+k`eI}lLzsW*g06l84yD;0IJ-m)$Qa=2%(s!giz3DM5|woU>`1TNiwcR z9@aW^(pCER0lO@begWc;iPW3?S|mpPE#6B9u}^-l4$$V9@89Ib5rqVw^-fL?GGN0t zPD28;QcI8kwnxO#*4o_Eh|m)C@FIOVi^5ARi}EtUOH+iQ!VVr5WKz9?Ak&Sn%n)Qc zpL!0YQ!=v%e}Cy<)5O}uttl~`Jo>>szUT)jF{k&XmvALg?Ltq)L`+TC18KPIV-XcA z$e@T(5;-GqU2P)x2B336auQ^f^66H@P@XqC{6PsgRD3-6R41n%Q0;tXoga6<;BP*? z`|!J`4*pYP-S_W%D*AZ;mVXo-=WmmCsS(TwPj z8>new_ZnG+eq}7A)TIVTN@gC1)K`Q}aJR8{a~g65W^o)4`m@O}!Tm-CqXG6d8o<=S ztX8CVLc-Zm*;<1#Nk89roy8G-H&;c5qI|JAEXOv@+OVgZe~w?K6`UE;j+mS~e!T`Bs)gG>4!&3FFBIZ0%Na7Rv<{@;yJGeonl-zbZ(^!K&MqpT!0T4WH|k) z^dY}#3g2XiSHJX1>Q%6L!{SXg>s~>P-tC?j-x9wNqtbNtJcCP zriD{l&8e@~z^6l94uvT$G5VokY%~vV^reRrpO&%VqNI&`(55_<;yJ3mobgk9MiEzK zCC61ZRW>!$k=zDlD$r5NY4#cko2e$CQOPU}FZN}7ix4vdw#ILM{N#lrH`u8qlAzs3 z{}qa|>{i?~HhQbu2cygh2JVsCLxi7}{U6x7lGCjcRvT9KXuQ6kWT- zzwUJLSsA~Y_tKosGwU3d9Z-s$8T_fW3+g>l56KJ2Qj#KDZ0RrZoza(GOt(#ZT)MFC z_}*wgea-HUyo(>I_JV#j+&Sb|VGZbDl@9pbIQ8IanudA;HBqCkTzE-BlAr>Hjh#fn z$%7;1iE%_mfi!FG_$6sc^*^9x&_c9CE|1z*8;3bfv8W~!lgbBj1=Zf`{`tVRZGo>o zQZwG{%v>MUZ?!nyBs%UCzrHIQZBE__9Xe34@c!!-@h`vlcJGjG+ZTs#`n&r4iT|9Q zlF_2vhQO?Az5-VmmxufJ6VT9xs@V|t9-`SQW@4v-set7K)LM$V=#X@_8xau1Qq-dE zv%)TAc(yj>Sr&D(>8W8g)Oyrl56ur}0{bMJp#xBjE=AN=owhjHnL&)i&g!{gT0N;C#f-U5{` zT838V-?;tsi_5^e?lkG_`IdRNj(+uVm1<(YB;A={WX8Kg7yn)Qoyvv229=RAD^#?K zU?4ZsOi=eSemIO+1waMkk^3L2%FGHDsF)T85M1CDh_xAh;+UYtW7l{XMy{PTtOQaD za)WRNaWv(mDCw$w=QsEY#%N8K zqh<9Nzo57>x~siA+rWRmxN_nnCsJ=1y?uZgeIGqh+*HXY&kYoYqXEU?a2W9OFK3QO z-jnCZT+)}G;}5(CnwO3YKN@aeb9}Vc-5_RkIh%$z@$*Y6qdOzr2oj$xshs$uLr}3f zUf3fYxTsRUjyWD+cnDd^^F;qKHcz?Nl?TKGK8GNU)4VCBf_2K}l53$gOOyfbjgP_K zL34}*(M6f)Yl-OnLlM2QBn|u;1gIchlJw`lk+-{{cg5h@n%ef@eB0pJYFC|feY+z( zxQ-7OmPc=B=_uCmJMkxz4Xc8$vmHVNC4Ej<3v9gZ z=@~x%PRi%h<3%3-rXFZd1n+W+7Wg{JmH| zTgQ~K>Nh5jrFqXVwl}tB?q7L8_s8}TJ?#`a>&Ivv&`sH#)m6}CYPi=UnUq`0m3ah7 zRatBo0p$Z*rx4dcDYeN%Q6>`Sp|=3t1U_eKc8qjS7HWZOLai1@pc`&wkuY*0lw$}u zS0#<(VKtCwkSQuZJN;9a@c_*K#LwW?Fx7+@{}*KX8}xGB38e)h#U8!IVwt6A%3uNp zrd;?pNg6>D5}nVT+O)bRzdxWc&2Qqv0>Z6l1$<0uHFl$SrsIsIbsxRST7;$ zF?i;2Y1fn(MG>Np@i9oSJs;yy^uSH}Tpx*lK9=%%ukv|G^nkP$d5zd#0q0bIn+nNp zwC~Zfl<{|(1X*z*72>eDI8FVRzClE!7^NOtCbc>J!@|Btyh0gtOX^2P7DOYf3;Z|zd6 z)oMwteYa&>Em@K!%a&w$lWb#4US*7TY%s>y##`9IfZ2>$;s7BDTM}bJ0wF7zKnTeY zhLD6LJRsyH1TwsY*y`)|t8;I+TC%(_GvELJzHc%{x6kc9r%s)!I#s`_x&^0+9l!Zh z-=RY|OrY@kUFA4i>XL!92fl`yc(v`_YQ6UVSF7K3;*QX}HKV!W<{~HatXqF2(KBWP z2MhMNtC3F$V}jNT7z5T@55r;U(&M$v zzT|lAf(2QVVDMOrdcEq$jgq)@?SiS#7gnf$<^ON$i|NL4m=*I%Er%l!^d{jld8s@o zZ-;Fk5bldOQf;urGEk(ROtCy(ew0LVe%w3&7f`()IEV!?n zVf1yiD>>*%@IoXd&kI5Xb8L!?O0;y%=IRM^M};6sfx8-AL(Qhx`N8WzlQGalYwIGS zG>uM^GVqE?GRYE|U<#@RToZeje)R0OA{TxE)EfO6Qco3SI=vjJx_Y9VeeL>v4$^s` zz5*3`QiNh(eDf^rxQy3GGqxWM&#CKLFSkrTv_30(CuyDDiRk%M!OnT@;ku?BTpwt( z{H<7EY{fFkj=5{X=f}o8haIMk7v6%^PkM!a57`sibJ!F5_o&0PSMle)^qlm~nGX!x zAqy+erRJ1Y)WPX-8YHV&7D~Y_wGd7st#|jBhbBCP--}ZB;2Om_w=xTK5v+a-YBgG| z;xhE75XBWpf(4*y$V?4I-;UJ8U!<=`h+RY$a_m>n^%^!j z2gx+GRh5*%mXnp{bGsb&B&!LPCIyu0)(buI`U7Lk5NtWe6OM1+3y>7_7OyG7-WX$! zOun`6#qWyIHU6IcEBYU8TkzxIO)ocZ`2+hN``*y%RX0PT#*C6$r+B=*cSU*4U!S%3 zle_P`X4afC^yQs9Fk_W?W9h2v@5cG6KJ%I!frXaCnj;o;wJ}gB4mw(t?H2;32E|pk zAR|IB0Qv=&2|{cLzw5DE2p&07aI=ezlhAhbG*bIe>j#g_M?u6j0o`07Br%H}zSr7PugcYVDa+LkfB;(@P4-)Qd@?|fv%CmQ25H1;?6cq0BeeuKhz6r|4*vhs+KL5DdBpg~VSs~R37UIHQO z>&26^(p%PSR_`vWVNcKK$+W61Wi{f|3|w4xem`epeb3_gcU9Ik7j$|T9KjtI-;UL8 zIP;qFAn`fuv3m9|;&l^v-O730jAd4%Iv$uj3Auz!L52Ob$qWVqv*7TL;d`J;k!&C5 zdpahQKu38oE{q5b+MLo;Qu6lJ>>F|KpTH>pky71;l@RQIJe)!n<#~(lU0S%fi_Iu5SJ%#%c5HLupxc<9oE03{k9j$f&$S!#$`MvXtY!l; z2w>UsL+uAoW`msofdSY?5)MMILwzm6KTLwRf>Dlo0IoExA+%OPi^c(5B*}|Fc^V~4 za|WKxcVxO;3Z+=UEgO(HU^&m=b7_$PqlO^@Qbyc;=kf(_cCvydYm)q6QD1$sFOptb zo72>mDZjk+mz%{jV?~9MVo`?+*YBK~(NLD>%P*+zCi?Pq7(AeFStOidF+%2WF2Dlx zCyE?jRLSm)%}ZHbeSMeu z_4v$ovy!3(8IG!gioDc7s2&a^QCP1ol5@bH8A72jjP84xKDP+_JC6Y%4NHMySh;Y0 zQ*AK=Gaxpl@XN?-YI;K+V%AUago%dqCJuo^GZigbJo$E%PNldNS;+*~i5pDxA3h-3 z+}R?oT~kE6oTENX{+irnmo&Y|{WULSI9HBzulM=`N&ik3@ubaP-Yj-%E*hy74jL$1 zHuc5Y!}s4@)myRtafU!1(I5S$Vn>R*F>gY(-*AEWVO3&7E9i& zI`-Pt@7#3LJ6F$tU~hrd!>ny>zx(!mKUlo@2lrk7%(k*NEAv-CQ2RHVE>8aEGjDVcPIV6^AU0+9&MT1;SRzHH(guO@pwfI!W&1=`+ z+}4$6FqryYPOZDVsd;l%=1=>~24h}_9O^naZ{ESKt}j6BFjG;{flW8=-hJcMJBo@- z%xVaH0Y81_1JLj)PQx-`N@P-w4JQjsjG%!6-8`H?PQo%kj)lXx;PFOb*;!sU(jtpl zF;&n&y-F=%nork%4%$)@4x=RM@(U4o)7w#>e0$mjNjx9>z<$4G4emHpgZG3yB80@} zgocs#0CJdTIEp?sf_QS|J@|*IPtBR5;8B>vtd`z&GYhg`5^%nzhCi9z0i@!FR`zxNOyi#n?9oc>QHbPt z&}E0P6aBPN;w%E`GF+;lZNT+Z$R4j?mz>@Lr_&RnC<8O1oe<5IJgTH*Kz;*%q$lv} z59O6Blak+g{gq!8HTc-KQ>T`Mw{6(4t)yg18rve>U`jSb-+16qBtD_ZQ) zfAw9}(tcI9XylLhne>6zf3FM)xir^odlLHDunda?fiBJhXHhzyNY>dHr32OIEl`pm ztBrXYVL7maPZWQ1wkM+2R*S!HcHZ^{^^-{+mYX`I^|EzsQ&K-sDixdh_4O-XXK;>* z6%}HN`jeUci)XNU`V-RvZ!`e|vy;aK8X!6l7hzw}FzSdM+^WY8#*HyQj1s!HQoZQ2 z#uUOY&%ej~L~Kk#TtWMu(J`-_a}323wD0-koMUh|*Te}guk@xNt?YGc45d#WB5~#FLT;0>)@OIu{yC{h~(aqSNtd z1WRK5qWD}Uc;{Gb{^xw>-T2&MzXKX-Yd+^YedFJub)EkmHXfzHQ23tXW9iVHJv5SoZ{h=yuelcD4#`ir6YF1wkPkV#rZg55_?@0$-g#m% zNGT!VA^SDq5#nYPe>5sQ=pnoRwtD=TB6nQ8*^jr?QeFaP{RvW!CeqTwq~tX_9=v-k#o#?P^8QFih? z67AePs$`rMqiI6!aC{8KXb|_;-rbn^+!BdDA0G3(H~xIbnCG`bF=*dYI_CN9T*!z* zC1lKAU^@zd?AU?48-(C8lE@7VgNp)eL90=e;2ai`5EKT$S)#-eZOZ{1=G(|Yp#grI z#mA`EvNrLZ*eC*tM!c2f;o`zjex5%!;PN|NC;@AyDw+%)ZoXgS{MJMVCtvOqbnZdKrV}M;#ndrhd)Ji6-R7Dw%bnvHZ`)FLlSJD+Cj)wLgccj;2BDI41C zkFe(gLJ3vNhQA*`SM+Iz3qsqpHDfo8F3m5ttSPB}0Uh13rr)j?ayVN6X z8#m67D;BR`^^3)&KXQsvK>-Tj{Al#QwzRd)9$X2jfps*USqeUey#)WtWf2!@YSG4! zSrKO9%pxXqYJ8iH8u%~-H8X=F-Gw3Q*~A;|q{8;#pU3et^bxiWP3D-8hkPa^WV@1p z=hXJiJi2eAyg1Hs(%984D$NE;o>Fw-K9u|@t;>{aEvgCmc5fvD3`<~*#d^$ zh7n;hx}l{M2JJ9OO8qt&fs@!t^;2j8)n(R9uyT9_UwBSz%vl7$2)_R0N%hpAnuKg9 zA)IhQVak`U#})Lzf?@ai7j{l8PY2oU%e|i4cD|62xu|p5P09y*etu^t`j-pHQcjkm zQ(k>2)D)P~uf+m5UC2fFtT^4a4q)G0@ZJ7(+1y{ypciTkt*5o7$MO^(NU9 z>M-q-HU_eS%Vq3eBjoG?^ff9%{US>W7}E-DO>FCOpf^Qo%E}oPmZ+jYjZzFCws9F? zeGTmexC8Y>j(y5E%}kCOUw+I{BWu@LO>n{#C0mrYYK{9r02o}*_n4|aHRGXM7J0l| z+4|)VF5UL;2OH)D-*OjkZk&7j=4SEl52^3Ib#>MHCwEC-A@P)GS zwfnAq6(7Ct5@gLex`5byR|)mp|EGKI`QW~u_Mt+n$$kr4J!y8~WlyeL_2l~OS?z4? z;S`g#aQO^z_wG+#+`Re4Pj+AV{>d7&5wtDY`19ZH|CRcmyLYpsU+sJK)qP8BEIYa8 zmVqKyF;5PzvIKj1DKiP;R?@mhcB3{~)bJ(zb{)sS zUHsaSQ_VX9#n+!O2Q}LKn>qj1&b(7BHtD8&C99?3*M=(;)yV#(l$FuV_XxWb>o4Q> z6yFPluw%3r=no=%YwUC5)nlKl!^Bsy@#xh$QyU+9u8r58gQMKQ)DRy!q3PB|tSE|Z zP;znCkBN*tBQqEjWHxey6qNfih!YVpXxvvxYVf?Zi3cHqJUVC~Vk5SICFaXBYX$SP zTt(DtgYT3|in%7@Ug;zi3k;PQsb&1H*x`*6GV^;>2(Sz#GHc`?WKt z?QvbKoJt6}>x{h?KX||UTiatpy^9M=jbF~~ELC6UXU~-Gh#nSy@XEcs7s=YxZ3jl^ z*O`!tx(1cXBlr>Y;qp0C0B2z6f`t<=iF+U@_TwT1>Y`2oM#FeQK_Vs~LTfXdgmjj! zP(3`Yu1h6NWb9d}6Q7+?ot3#RGvED{_))z1O5Z%?kxE~>KY0P0(x7I&R^0VW^pv&$E=Tr#On_;A)$Fl@;0hi)Dw2QJ00EK* zJjVg0B7DU%M4sltcIz%|_^?NJV7w1odhOD$t{N48|7gBut~jrf9hvrpS@4QJ17>ZK(wK8Pi2`z(J8`|`7?uMC`eoy%by90nkGtTHq0(5IbPC`{io5j zynSz4`!?y*xdkP)skewLmPV`M>p0GSH){{#4kAEMg7kgN-482aFp?4u<)m5Q#g|d5 zHf{zHr-G%xd>m_}w{k>8N`OeiTF&!1b@h8TkW~zQ3mK1O=m7BV#`?N?tx7K)ehRo5 zb%olVnT%;<`y|S{63#8!?bZUK4#eoZaosO$n4s zHzbej6Oqo%S|sa($w7PB+`8i4hSgW*s<%T=XPA2L$!=)1PM@2fzJO%-nISnOZ^QXa zg*B0qprL(?=?$KpB+eLwHxB{m3myNEuTIn6`s%b@8P(o?MnI^EEsBvN?4BWY)j14{ z(_fW8%0G?R_}KY&PcUB!!2C~IK>zJIYc`Q#x$-!C;+h_wB{#2# zJQn-hI_0d-F^)c`ab^_g0!_kdi}XD7uob5cJ<42kjMBYH5;l{>K5!ZoHUn6hYShAr zCoQ}JWe1Swp>*49whUW_)9c80QaKl5%X+3+f2@|$F!a;~pU)#bzh2G0rG-6Va*-7{ zvT1qvrjg!s*ACjY!o|`8_$~~+r`Gf&DFyQ0>D{M3a(Tt19?UV(W^IkwpxY#OijA-Y zx(mml9VdoLRuG8VQ5yG*80>^A-C-vy91$_(c7`O$pn=HhSvB*?A+>RZ!)<(z9}4xJ z^=$P)*0~u4kTUGz-so;59HV}lP1-%O7e`%-GoHjvhTDgjlk83*vl&quCLe(RA7fDH zlIBE26nb1>Z##FVkNI^F$W`s~i6_-xN32Q89mi#>t?|PdA1?e=r{q9!O1p*aQTMk?2}9N(IwXK&LlAmZIoUqA6OFWOn#lA}ReWq4kDDl@s#yz#4^b;% z^!eC$3JS%RChmBP(ZM!Q^l-7@o9nv1+}G$@A;3yMFSV zTNho$`4`O-#g`DbE(G>I5}BHZypIx>5qC(e*8t!f)Pq5g40{y1T_Ow(KLMJ>@kCdm zi2=T!{6JnP6fjbet$6dPICx=n!5m=57wq*)xO;*MJPIPdR5z*BTkPvzecSMwyH|E+ zPVSFnlhqGiYzAtTuGIWr0g;S;^~eUUpmKncURMrL9{w2eLl!B^GgS^Qvu^s;^}= zxN{q7bKFgSc1_8$IrB4{>@7QH-#Dl)ujy(myJXttSyh9j?8>c=>|dC*n(v_dGM(QF( zq5MF0rq`X444iTetFgpuGuM-v#Ns`1oP-ZqV)}B8fijT@1AU6{EjB)+VJbN7*uKu0 zHgCm%V{%p~r%04*nwpjc+v=;A=S^ucNMXs5omSYJXLg%{S*bHFNzW{>T8lCzv9;7X zC8y;;`s7F;Ib>~Iv7lk~tQkW+HAxL#TY2I9iloe>`fznoxVt6S@lk zktOany5f$%uOnqR&eL-DM)eyNiz=43ZP`*CajExjW8YtpVo^RkWiX-Gf^E3?}wyr@sD?GI|O+=Bhd2gg=L%JWh^6iqG8&T@$gVav&*0+G+H9X<4m zt7>VQGcBxJZopjV>6a9ouOHnA7E_%y;8wak>4#Uo!<1yKmY*zogo?0u=#MDsP%b0)67Y!87&oai!)LWFcoeTnWY%jd)aCu zs&iLH)6n4W{GJV9T(BxDCQ8r+XFMi5PBv zxbP)hh6?h+rEbLn*Anh|)$U};7}txCoptqP@JVXvnp`nD#lqm4o?}xkSvkjPa@ahf ztZ+km$RU-sb$4ZC}6khapURJ8yg?{&bJ?~FS&eUsOHl4T`MYfY%8Ap z$TRm1PHPJ1J7@MZs09DU{HvqI;t&*AAyv}I!_>We{#{Q|qB82)>dSy20QQPsBLOZb`r5!ZF9*@#!;&qSRV{OLd!uy|@v9$j}GbZ~UzU3yw zP7MMHN7$9Xz-#~p#w}z=GEfM?gtnSsNRo=0gJ^rE%PA!xm3u<3)rQ&XuU4x+?_txr zjRyATKS(z7OysM3l1Kg_*=#j624*)z9}+Jy*_ok+evfR7nUH60p*dnh?OO}L`bJo? zup$Zl4-~tQ=#xsqbxsfu@~0F|B7YRB`et%}loG#7$3&S2mhs2xp%v`jVJb1_LmrXI zw4C*?P!Fd`cauCaOVL^4Q%I^e_4X!@T&mq&h$cqDSb!$b_iwSKXT>wz$dMqr@i7#K zK%Zhwe=fuG5lkvo=!v8xP*A&*k_QFJOx$g+-4!%fJ~!$Q@(;*Lwv5tyVBniXa_cyf z$CgN|Qku4_wMW(P4)*6ANJ?YFY?0zh-puYgeZQExF8Ut4VyFJ5xZM5y7M5Wiu}L4> z5O3%|6*TPd|7$#0moJ{#ra}GN*pCa zGLIbwu>QCa^acBA7`J}NvX^)b55mNk2qj1)LB{}>!R$k0&={H6G`aFDkS9cxu_9SE zikGSZP92-fc6(d9T8dLk?V-T7%D(lJwjP*Nm{rs^Z((ycV2p=R@>t;J%7MTZQ(q@NHU)>7{sEpJ1W%3WoHcO5 zTkNuv(Ws0@rx=90cFi${VDwAQ#}f9LO;2yXeCdYSjad%|8VeHlR{9bAd~*vbrde;9 zyNSoDfFJq>VuHH-5lWPKG1+D5k3SD-&*5(=F@6^N+%|I7=NLzy(>Tjf{3{BzYKuI8 zadwoZUl~bC_d3ilf{?JH#M~Z!Py!eYWe>bFsQYvcGVmm=-#JGOucB)~CLTg|pq~vy z2Rae(E^M)nffM#2-`DN1b4Vim61e!lWrH5LxY+tgti-%I;^b&_eR9rGW>}S-v{B62 z9{tYIE7>nKA*Z(V8nTlNrq-T+txC!^7%umodi{O5go`+h?_n!8AvWw5W+VAA#fJVc zXmLxs%cU_)%)!~DQOF4zlg1?MV$UKr5JdZv961BubYh^l{gpuPnrv+;-IMn0SeLPLZqWV;8cS5RlZkUyIL}e>htOk%*J8$ z_#nIg3+#S_lF|$1hn%>h(P1eGy3a1{9JxCgnior5aOPd5SSi8Imt*hUPH29FCE?jA z2^aV%lUxD_gz{ZdVp>XqQjnzWpdQtrc41NQI^n!X(*C$g8$Wnp;6$O@RTjfx8(EMl z@&LMna?GK6=nWsE!aG##3I75&-l4o_t`;LG^8fi<`=^>jUO zXr8LJu3Y6W3fXtN(~q;4*X^j5Jd3}6Y^wU~?<14?FPqP7t5M{@FzbX z*xw(v+6L9{XBDL$`Qg&T$F=<+o2Ui+5)l?^?2tQw^8)|^QFDH*HGtEz<3r#XfvEv* zrx-!ZI|56kI9%Wu?;t^paTLML$VZUi9{~a4Mn36M+?iuO1EsDPE|S*8rRuw78Fht) z6)7d^lhaxQJ?35jWac)vtyw;W4U`s`8n0NiXiJ?fkk8gs-8ckKiEL|LwSRH+#ih#v zg*A}^H&WQPUcTxP%~8^H{pRSdxl3v?O4rSvc1dYY^Cg|)TPyF>oF(MHCb}#Ya-l)i zMv_zE5h=`1wQjH7>MtFdenmLT z>ay>G%LcIbzHPVG*6%)Bh*ha^~qX8&J8gHd1ARI20l7C0I+du>Vj90FH}@ z951AQQRxw45>??HZVylkPNN|!Y^3mHEJDd+4b`}!h7hmsiF{?z?Oz&rsExTA5iK8n zXyCTX4XKmfe$jw%`BQ2y`53w#@+b8@DfdrnA6 z!kp$f^4)w+l(IKwP5=^ZQ+ ziQjW1_FVXl_{!+_8>o&7{~qDDVwd(D7$Vw7{W&m12%(V;fXS63M(!7uM(hZj5i1M1 zNfsdaVl_Kcv9Hi)c+P$Oh(sVILLP$Y7@7E^l`?KA$@qG|;LiwR>v`SzxDDuHpl{3~ zFa*f!LdG<^K|QsZy)@`>%8xVC;lI98IJkZakD))BH8uZ?aNFP2zw;m<$FS^^gs33? z=GG(idNh64u8{+x8pSmVIEOH;75OliMqGI*;L&s+ZU_?YS!At32a=9S10Mk5z!`M~ z!54;5u~r8;ob}nt*k=O~4-RcE8MiJc#W-VmA5>*-z)vv>Jgwdf4aZnB80 zf>hQyyXCviP<8$p_3wwrtobi5C=SO`wt86orL}%`I5?n&XOy+p_dDy>Etd+S;@IVj z;!Y0Z>9vitD3+G5i3pHvc5)@)j(bror z@1aH-HfAxJEw~rN$wJc1$g7@43gSbFU>HL-VZvK9kgA$Syr^hfSy~dpbqZMAe17mb z95?Zhargv3WdWRg$GJa}&C{P zX99>>3x1vTlq{8q+37;@Qagd@Vi7Q~dO{Ek{DI=OH47QrzrOkfN75vD04 zHl|O^kQtU>IPuV2Y+_n`j6!5R>PM3yiq||0Q7l?4lYbNW>cH)f^gq-Von0z3lucg) zIsCMG4?G)kDeA%eOP#}>RDo~hu{ti3Q-y&@lFJU@EE*6(BgCla1Pd7n!7qs`!CJx~ zna&x+WC9AdVUp59&WMBhpk9CqutBLps+;7t%LJ0TYAKLM6D8x1qKmWy1U{WHsjdDd zCbdl&QxV)lI8Uu%uOOrcp)LE^Zo+Juco?eXVHL8Wywz2(9;xqbHttvYwHkJ#NB4?yPlWP9 z;Uy1Ux$?@%X3G@(^Ux)QYo5F6(B}HG^aiUX)m^%1O8HP%Lq}oCS5gW)8oGwcrwo+2 z;rMMxFRR~t=&I+|uwT!9;_ksYmxe-@&Y6GfGfN)$`*o|PmA0h2_Mj#B&+gvv+{Ttm zcQzCiHSD~!W#e-j?*7@@WZNEBdQ0iFRR=!dXA1d=%9EgL9rv>!<{)N*P(0LxD+Zl9 z6a<1>bqG2~>X*6;eFdRZEt8T_k8i9~TU@A?yH?I7jwc=ot*aI9d(qw5+qI;4@{Ym% z?;Hva92;Hzv4P;BclIy4YI5`^*EM9H0N>9gDSYjjZadE8pu~#OET8t6QbwOACl+Iw1-D~eEDCopL*5J&h+LJcyQ@0kYh2BA>KYh`%FsG;}SWtj9!{*a7sN-wq-0Vf% zm*Ru^Wgx+M26vFo-F4Eu^2N=q8&Vt_r#CODnET9)H$2gI?d0u?`mbngKm}gA(^;98 zR_=79RM*us$fnjSXHIWxZ|S;n`hrI;yX=w0oi{ICdP_&=#>&abwPa{0U3BA9yn7Q}}`9*Wd?E6Y3(>KC6i`!_k`zYl@q<6mKH)mi`r`A@UN=@j)Cp zoq8N5!MKeXl2CDe`L#K(488N#&|9-vWslqNz|YtG>JBt@o(I>kCuQWH;+LZfC38wm zjfwf9Z-~328^mcQy9wT6u9LuXBd}-E;4e zo7i&c2K&fmQp(6D(j9ih8tu{B#J8d@ai3zfXgFf{W|s22LeP>h^QfA^a|RP~^l+CN zGv?Si$51|?_C1fDbBsSW#?l-claQaN{TB7&;ol?tMp||DeAHnh?LQyGarLJ-y*+2% zF)db33VvZ%#G0F$1lTB~e$01*DdQyXVa=#YCG%L4xxYSP7~(-(zd_-_ zOQ4U5e~FffhF|v!ez!O1<0kW{+my_NxXDaTGQ7mU={eTz%TFq@?!u+gN(?$V$2B^a*M5jrTWq)=g?lJ6O8w%F}Pj z8&BV+^O+hZdLW-*2kd4>{HnJbPQ;{*zm3_x$?6Wjm{yCTC8yS<}5$TY9JO=$kgD-2J?}yl-mX zj+WldRo--~ZEDt}rmCHn9sG|y;_d4`{`Uh-gTqBd!-GxRU;f9Q->^9^G%l^J@A12i zBqiOfzW>X6&ItDo-12Dmq)FY6-ZF466YlvXOTRfO>4?kUQ(wEZ@ud0`@tg!))jMd_ zuoSjywvZ=06|twG6gkeDQAk4BK~8RBI=^5<>b_z$4GW3|AS1 z(u$Q4!+0_}0HfrsB`DA_KE|FPVk%wTDq$ZS3Q($XJY2RnN9>8@`tdi5vjUGm`R z2j#)D4rq)UPklmXQ|w&1^39bi!IQYFLlsf}dvnAU3I=jBGrVXML%!W)O&JiHQVSPy zM=nn$!Ob`3$=yD1BGa46-Se^Q$mm@perMHT9f)fo&L#_LM_aPh@YGL?hNLy>Rohrl z{qqj?y&1_#hKHXqnUee2rVZ*}S@}k^ic168)Klhk)IjXDvt8_{lVB-n*7x2sX8@Ga z>r@|B+Ylbo^G>i9_>~re4{%?AE9C)UKh}oY5EHB|VG(=@Hq?gDE-vta5vCBgmU_66 zv}jxmnc^`T0rY!7{bLC(a$B!eaGg|Vz4*-@7A&)vDk`kMQ2%McJX0~xFMW=A8a~H7zchB9FMMNko(9wB zoG0bd+zUUz#nO{%ZGfI|?GruWPCpMl?^XX$j^c1z_RET;S?zxD* zdH%3NvX>i^4abhjc4K+D_1(`b8dL<-1gx^M((=0c4+>-R+z0VX@ZDYPJpYI3i96v% z?fb|L$PfBE@`FOcRGuGH707kL0tf{&Gqe+EAtR9j#^f16ojF{ym%&mHg`RW=&K%iZ zJWc4Li0t#J@OG7;YV5kJ6@z)|-)4UOt6j`fVlv8u8&KJ{_V?2td#+X9QDRIo+;NAJ zY%D3U{OUt$Dnkv~M?l{hpX zzpsjC1!-=Gcwp&V0_o7FZiQ(bgGWDs8lTClww4<^;r?|{3AF6ME^Wr5H{Qq&8 z8x~eX(lnV1>n-TO#V@4C)p*@ulD!^KgNOYfs1!SRWopmyw~Uynf_!&UGxDH- z;-Bzrn86|75rP0;tXziI#((j$`6MnBhc9~B?20uyAFP)C0ew>=td4j}!-ar)W}rBv zJ%uNFW8c9Ev}!lOo5@2A%Z>@FU=%k}nnzmvH-*O}Sto86aPGU?L`=&AJfn#SwY)k$)vXegha z6~4S1n2b|UPl+l^OdX@LR;}LETDoPPPLr4T*$AV@u&*MUeN_5KOcLbt{ z6H8~H2v*ARE!3`_(j4d^IoNsH+4g~FvFmwup4k*Y3=krBEJl99RYN7O-g~&|;uv3j z>8V>Tdf#L5o<^R-pvi<;GERV|uT77|pHuDwp2LWmW!Q^3a5(H3>zjVOm7*6xTVsPQ7C96?dpPTwgB=>al>a5gip6CD;=hQkrM21hX=wEJ{y3qB;y`*c87Z&85ig_geweAs+Q7* zKlC0=D8ebsad_PM_+OsMf_@QPhK0!CFO`X-Wy`ed3Y<0w=_>BW_6=_eUZOtt27BX* zoSI10-hw$_**Cl)xQunap@z5S)HYY_E9h3=o3r-D?JOMqv-mHzbNSM&?@cZ0TX*f1 z>Z_tVYEW+-T9yC9sUS7h1ed5lvWy56%I=vuPDl-}?EjovBmaxSmyZXIE9sORL#9zqP|R?N>Lb zug%PQv53oo7@|xw&DH4Nx7(d&^<(u1Mq}De zb@onnRK4MlC=RC?ji;E~>P%V7zP4Z8&u-t{Zxho~*jLqV3x{H)iM3(}+yWMsX+sY7 zPPG3@?(cU;|4;P#wAgu)N!@8|!m{ zHAMPUqaD@S$k3^mfh!#}s$z&~RE0uRBuUXQ7c?{w0LcIVv_TnFsyU#g!f6>BfSVgC z-x^@{qUQtjs>R{uRHZzVi=`@8te=OnKNxvfTKYN2D^6R(WzuT-FW{kCVQa(}F3iqA z$;Q$E*RF7}%9MDaQvlYK1O%}pqka)&QG$Za1D2LjZ1yCKz~*RbmJfxRMJv(bBE&06 z(JDeMt2LgjXfMQSq-c{x=}`WvpfhiFUT05seS`Y*3v%6(+J)&e_Nl9FN~feu?MW}$ z+0TA=VGev}?&D3Nl}kw2k?dZ=`6VnYk9b1)=|0dYT$H0}T(pmioi)IUpldlH5Trqu zGRC;l&B1@^382Wg#pigRN0#IvxOZg`$V@Df$Bi#Gs)a8^5}&LNfAPzUF8chTW81%d z4kn46IpY@a2tQBIwIzx(_T1u)KL?JDp3~>X6ZlsG&03Vn_*~P)O_3DnVxOjrEeQUB zuPFG-Va9_z^&L7nga0iG2gshs-vcM&TAE!zOFzoy%IRFt^hBRK#HLCEyVW17-$x^m zEH0Bi5c|ZPCR3_J5@zIoOODk3f6Y(pm$pUSMzdQ2Ff2{lq3yM1L*!E}Lh2OCvRo<2 z(8H83FmXW)q(EdAasx__uEQ&)hv7$srtUbDi`J=EBvrMCgZ;G~-7VkEYpP07kL#r} zKKf6T${1qnDK)aizj0eQSlmIO_q^dvrP8ig<&5Lv60LFu=grh;q4*+v0e+gVljfTa zK){9On^B+`s9XkcLPmo=U3fba97!7XM0~OBjF8k=T-{t&9lknmQp&q}an^f}p^iq= zUr|TnrL3(56(t$PxwSc$7K*79)TEjkcd~Wyni^>GB)+H&{g>tlG~$1Ajs%mr&?I5c zu+PNjFvmQiBeG#pDG^O3FG$CbRw=ES7~%UMGBs4^BMOXegOS> zb9_9_hx`EA_y2~yIQsn@OQb#jEt{6`oO&{8zdM}xJse}CJ?~W-_&MQp!`+}2_R^e& zLL_Zf6o_WU%S1D(F-tNEj>^bxHp|Ld3qx2-FeBMXVkY3qAr{bg5`RgiApu=4=HN-I z&{{yLPeJ5~TG821;c-*Mh%0>+vB-B_y)juv=-B72!<}k0-%AH}k#Cbnkr8Cgmu^jM ztButArdBrf<&dGx_T%D3+&5hLMmNB;Nrs}ZcY~caC4!iuWnJYT>Z%5J(*KuL_80a$%8T~W- z4n&C3QbeOP4Y|cNuyJI{$;90ex0JPz@|2wcC?Sp}8^AxzFof1a%rIxlWLHq5ONE6~ zUypGKMJpjpI+qhx4g=HUz%2b9KDCA4bhbszw#BEYm$H0Qg@5wttvxv zBA#mn+mOx|=pZ4J202FT|uaksAuM)M65ykalH|c<)IH1~J ze_>4^{d1&8db*0#L46lsRyBmk%4@AZTOEj7*E~%MVJE?>I2J4K@~bCS2--a}2rON( zfIKMNWC0x)nH#7xx=K=JKRf~uZq%Y>1ut%iEJY@yr&60Df{`hRf4L^oU*o`XJpA>` zaVpGX{NUrNHCrMYVlKC%Tz%X>P#5giA!K~oyKUXw2C4w{q7GHu@mV^;{uPv$N$2M$+`wzMWXlEl&N#w$tg93aLomGZ2OtCC*DXH*{#q*2H z%DMy1S6te;p?>$etfm~rs+@I5M_#V#sjIJ@*Ih&^z1&riIVY>ce)k>gmR4_^Wii>( zlFmMvr=DP2b|)>VEuX^AWisUY4)9~HfV$e*Y3Rp{+UCfZ#r`GchNY!SW}XqmD|~Y( z5-|Ov=7*jIB=6Xv-kagw7kzuLH$(l#UiS9tOiBKjSqzSp&Fo?6I`_y4^}%FZ&*sgu zh#ASOR@#`}*JqIc)ZHg;OG#t)K1_?`1$6pq$fV8b`hXz%RgFW3CpsU&q@Ye(^uMKg^x4-ZwAzKjyL8 z7MCa;{hDmGbg~qs!loL3iM~D6)dn&3IWfgvSxHH50MlJ%MuUS&{7{T`1*pto&xel; zS2YEVqk%7tne3>Th-Wwwj2Dr?1f`hOo;~&>ECF|AS^`F34Y=85x*T4w!>LseV1AE( z4DsO{osj;=S+9CuCidX3dwrSe6RbjgWukp}Z>9Q1!bXHwN}kwG+$|33J2AQs`KT|| z&a;wDM#Kjt!DJ(cC@NtZ(KXp<0z3xLc-&*D{|sIHQOOAx|7C<8>B&d;H;2>d$a6R? z22?FIO1w9h<^sdMpbX|jTl4PQf5PTGI?w<14_}^OdjeCw9^0R4=9w6plV5pyRWJIN3bLakl>& zM_Pxk8G!&YBBWTPmHqhZr>KQ9sNTMgUFYtIKng>my;RleV@q_%>5MnqW^PIU!^ObozkqJ4tqbPOxVm@7Rf{(U`rUV~4o;i3q6ODEq6_Mti7#UR{HV4uKhl?p8n_uK z*IZQ@%mKU-bTRsg8;LF^&;|4`fF492l(e6SE>kAg)s&YOhC;-R=5y{c&qd?R=x+*4~?G}Rj{YYdfzYpO2k2zQ{{4`d3kUOt@3pzyU5Mp-oQkOF~Eiy$rF2IGKI z#jpv61bHyzZo=W9TLoYw%L%^T4}gQ_IV((f0fqiXi!pa?-0G+B7t1MUY85iyNu;6af+(9j1pUtLvCxlO?J$-&_(%YLHX@I&^;*23nz zwA{sM^#vQRU3+B(TVB+Xmyy$-j=$3mUe5j=EtMX5eART%69o+!+m>8;`N-i_Jx_l&zk5asep0){;-ZL?!w4%TNA5x0T zpEyqy3Ezr1ilCfA1x8thGM-h)WK<114}(oahX0odhwLZM#j)8)COdT@R+DDm&Q9Kr za|w^Oq{0>~5gv@7RJ_{(k8Cy)Ljhb?U|pHXUtNG>>PPboUT_|1qNv!R6@i*c{f2o8 zCcGLto zDux;&VDe4%^}814EoM*m1*R-3nOhO)W6vxJ1eWv~= zDvP@th_@C7Ds`RbS6VczLv0~Itc^mC8&!v$sUf?`fS?X(>00zCg-k{l)A&;8!b)9B z;}0B&8H!(2->>xrm*nU7PVzOf^jcqNT4r62ua>1Z`h1Om_{gH;N3KBXBc4;LQj)P$u;J()T|e zy&iQxHeDPgP5sC3W7DlXzg~hoehEVMZD279geKw6h_9*~mA-^2d7_0)Wd@7a30F-# z%35HhyawtXDwvHZkwGq33rcY*76U0Ux~ifc34%o7^A5x}60V7cD-v$$S%o1%nB0Jb zz>2ccP*Y)(KNl0qsY-KXB%wrv%P~!=uhS7Apa{RhJ9rEZf4Te)A7oKIN&d6wi4>>G zzQZ@%Hb-_9ux}2!>_v{wsi&`rB{ZN}6t-uIYY6nr?P(BjhE;tv^CcEf=_jB<>4> zC&&b!o*tzo#1FD$V~!TagO3Pa>kn~J?Y8_t&Jt_v>d<&CZmq>_NrgjxJE!a8ow*_nXAb6n0enA%*s(bX)X8 z(ZNIeTt?9vOq?f>>+5ry;EPF5v!}p_=TI&R3Z~*eW3|-CXXgWZT+n zup3z}`^etgXAqMS%33Q~5q(QmL+VeAcGRrLUL*tCdb_e5{x4)Cq4hZsX%?emhVazg za$s~>78c?kL54RH@n0%xnk{4lW(T_noX5=yNoy#Aj6oE#3-CWQkJt;XzMn=$YHgBq z=W&C{H0`a%H+<72dr+2~&#+2cFeo#d^5;JpET&-4qQ3Ql6P>*^jFkKl&w+vekaA$; zC&uC*CKz+_XqGDW7Kz*~&#{V=(e zC?-uChp3R2WuA$r3LtZ_L8Bk@OmmOlF9`m8KWd5!xk9cFN6SnlX9-Q0;D?Q!Cyfi? zUn77%8=m-3X-974EAsW{JW@c*#EzZx`WlYu&}Bpcb(++Do^XqgdOS0cB`p|{?5Mi~ zhhu&~O}OSw@Npa_qVdc%Imp`7uTCgS zZXZ@RB?yyShsCvLNfYc3WWy}@DN2NEBd&ZuN=qdtL3D%)GO@97>`D+A3t0PX4~1^v z00r0%|M22d0Ha}t4TeUlBoaQ)I7ATu*wBXyNXp=ez$h@~OGm=2!$hE2eXA?rdq71| zx-IBAvtB^);%P&goMnHdEq^>etDwxWuzq$)>#VCgW~`i*2f&ocK>~H7dIQK5+xKSz zscDC&_EipSs!i|gj0@C+-3KYSeo#&;--6#}Rm28NsND)5o~9m1u4Y0wB_xCK=ud)d zMY9&DUnYS*5;p~xCya=T*q{*fd5hvAc0702!QFzqC`u3Gf|gi(JbPEpeBjz;NVskd z-%~k1qMl4xXoAiVuD*6&ZT0lDgFWk_zt&X?on_?e&{;dSH((tBz@!?HH_Tpu5~+;EPb_g5x(yh z)Q365?E+lHGttxx0Y9XyYZ{dRYz|<#uwHC1Dk}&AuN;f=5+El~fFRDe1}~HomjRvT zpqOAT%OzMeT8e_5wVb$EhD&_ebcWb%LklMr{k~vYj%s^QoVC}RYG#XA#FXyY#r|-Y zYN$gfurRADr$1|To%9S6-FOgirFfJIfcN(&pZ;Y1{m@N(Eu_Cp#iYN`8}hKG3lcDX z+L%e&7+@fNsl3iH0sJ}X8tq;56AYfEN3?f=A=Jk-$Hv6IUpq&-oz11B1@2KA=^GY4;cZ-j+=M%cTt&BG6OdmqfHvgsB7t!opwLuTN&1Twx=4a>uxK$$;Z;Ww2Oiwsiu{#)XE!go7{ct;#Aj0_Swqt!oJd- zI|{=q*=HMFsdxq{o@QsKvRlgQ_HScPI8cHYMzS$qy<%uKa1ZRK`9^xc(j2Ex5@ek7 zGG9MoT}_pA4lK>`&&Hku?01Yt5~>>96v-$>MO=tKHHO1fW+$jbwf}%f>wx53K^OKdBp-ElB`KnIX9!cdCljH(w z#|Er@12FwG^fUCYjjwjqv;AjN|R=Zgh;KY+n0dfla6NjNT4V6eHC90r# zOvI|u1~uBva3{v`ou7v^inR9O$EWM8)_Qn_&T2i2#0e+>*Qec080@7U?R@9a?6M`HeW!MbBM!{`L9S4j?!;r}$>Lyk3xyQjq z7>JKH8bGDA>S{r#o>o0=@}zpy*cPfZs@XiInh$R9i%#gLw05T@uD+BkG^jZhG+>-`%2EJ#)p%D^P;lyJ&6a+EtnAy!4?} zGwu_IithN^cWxOkC?PyS#pA30$gAo=vJ+lfI&k->--EEG0X`}!&H_9`wFfRLG?-O5$k1C| zE$@jwU)fiXs=lUv*s^rdC7Y@`p14PGxLO01Vq0Zzp*8v5Cp)S(U9xCti~1+^XEv*4 zcA&DLGHchdq}TuQ(!q^sRrekH_N#M^;=cBXGc_YWztS;lYIK*e@73=dyRRy3` zWj1ED4q0R2&jNC$Y8rgpn`1{uE@t3LD+NBi*v-kpIL{Yzz`7X&oBoesA%mJiDOxX>8`4tOj4xZ~7UOY2A860_B# zDL%=)R{TfwHFiI`Po@0Gu zCxn2d4}%Lv2g7PfSmU_S|CNO%0gdLbWpsNt2FNX=EmSOze#!!lhDf@+&_v z{})gZfKDAdQ8oc~;(TvKF8l&u6r(?bvtV`c!{Ev@Sz}3l)IsvBLqVQpV3_4smoC0- zceF)6AI^39`N$A`8-uwo0jjJ%Q#P<})2a26Ei3s*?65@NpHX*6_G&t;8aayji2wS9 zgJ;uQ$$-E`-^%9vzZ-YN=K?^m!Ved%&d(}Khn(p-GOscrDnO!L+T6R z+&OY4HdNyeE(h}vfwN>lUps)YipjxJxaW+Lyaga%iBe<{=UK*;3v3*bdJKa%pmN6Z z27k%8=OW(%RWJ0y*lGeGlExZ7&Kp`m0aDJfE~97hh_wFD;l-Drbij9W%5@e=72_f_ zlleBD+O%$y@UWP(y4jgk6A zk}bJrEX4+iV!ZJO9lR3F_sL%YsrFpx9dakqhG>3O{LY16`aI*Qc0gILRjHp)&{VB=H7Qu=t zIo2&EM9;Xl7;+oUt&QYkkPXW~Sx6C9tn(ZJ#E2?qV)bX>2n7*G25)G z+BDWIdrgj#L{PytU3nJ;c`0a=1FdtCkcC0Swu1^4 zql6=3D**Q^5@{MvbdVYW5_D+ODULb?y46-b`W}dP5wFF5DMs%LzXEBI913bQbb^LS z2uo3wOO1y0fkeA*JPE}`4fDgjM#E(5yQI48EUA@L*NXJZ=7c6SXV@pT+Uq8FNY9jY z7Ki8MENayiShbqnzp6T9clFdGY^mw5-%1C-tEs{dk@gKc9??S{)h132z;uPfWD8{# z2+C&YDvoG?6oDyEch6BNLZ*CdsB-XJBZ)8dFO~Q3T!nKCz3`|Q7h)DDZcBXV;e9<3 z-KhzUVLv}1XpQ0{xgihAxg^^EVvnzPw))%bt@V@g5+rV@zWu>w@K!tUH5)*iWK{7) z7b`1*51;_g!yv&K#KD57+=6rk*?181;3}OOj!Kd|q?q(#-x}x>jyD=C?YY2cTo*1p zkixeE5noD*Af$LwJZ@Cf#w3%82o^+ss>fJ)Mi<@8gO2ubOFvS!a)^~CEPJ{7(#b<} z94a=J+r82wT!uY>xFJpzyg#B0oC}Xp((?0hLSVja!MZ6-vu9L*#SEMEsNSNjp5mEg z>g_rIV372DkYfms|)8QCitb^=xccc%NBR}9!{@W zG_&uH{o5}?>EBTnkPeN!ZCkN__q^T4hN_ybWjEZoEJhIQ))ZBdGUPXK`a|$AULDCq zr4t!lufXXp6b^-0Rp1t@Dk@ZD1zj#Q=7!%4J_Jw$*E{kqf?{ywlQRrbN{5652*Y{4 zQ9N{MK`BF&f-R)-xAm;vgG)y(1!{E=Cwy0k*Tw?2U~&8<)e`&jXfDh-E_SIZ>>#`S zGj-8>>|}Xm+ve@78urdFzG_Fw)yHNHJh|u47ds2`sy?{!+JRNmADmONK&jif=E&h) zZ_E;N_AH2ge~0wJv5Y;#8?Ma_U*erqJb2H%MUSpqv!tfccdBdM+-qjnb-2>QbJ0F~ z=duO&T!%wIyaQY15O~KSe2-2I`36vkLBB8Qcm9}fV2nr^#OsEWo(vw(MK3?ESX!o zrAqs)Bi$pfxt^kYKmcy!nHjkDAoAJ0`#ft{iA~VRaD74j%yJ|sH*>EFp<~dwmzZq7l;`ZxN zY1D25)HcRlqh2a`rh01emD7F4>!uEm7{pW2|DA5m-XZ-Us)#d^ol;=Odmk;VsVleK zQrCiJLW|^Mffv|}rKg9zDId|k5#M3ooPt+)f9&ZajUL890i+zK&j8mPzdI>L13);M zX2-7SPCD%v(p~{_9DfCAtGJfJ)p4G8h|HzhJFtcDI~RT_Hm!v5+8u-pHSNdrY3b+F z3LpU2g4JH_z(3?Yy1BL)es0k?qLT)Z0gyiz2klc_E2+1WN9slpzXPQPc zmJGSi|2*G!bB&~#v-eqh?X}lhdsV;K+c#b=et!biSn*nrJor|p{EVs-xQi4{(+!n$ zA%$-d8vH9b3c@_9kRk)PD&(u+OnlUl3=~0OazEu-uv4?*3+2r~B85m=s;_qAvTSnY zMp~Pe7BlwL1Y-FlYF@YeOxB*##uU$T^D0{&MgtQ%-~B4|BoxfBaK7aTg3TJb4TNti z;#4pyrSA*1zfYCLOTOIPkA+hkQDQ|QyId6nBrl=ko#eBS7_X-(}elS zX=GCXkZ>pE&xzP2ky{0zUl9~s5kkh01zl4tp_Wi(gsAk7R7 z@qnQPfax@Ri)UqZ}gF!wAZvHc+HV;r~6OD7H4!J*5Q3E>V)B_tQ#6 zCmKfK=b>@2$fnOsv)htm%i_w?vUSENMX^$=-FlNoYj>$l%oPoXqOLqWG)jSO+O zm~}}ml<#HV>lcl_A?_AYCe(`VPBiTFh(+acZ%=lI0%#pfH7o`4kJC^OOnfa@YsWF|sb9Y1(WS)vn zNAG4?A zP13VF6I52qxXkv;ZDPO03USvbZWK$G%HR5qC4Bs+<6k8)hYj`M$w89qgBEg>;Frq00>~UurFkgq4ag;`UlGLd#B@6k-^d%k$U_0rd*ZM*a&&fiec78uqB^4T4n^SJSYxv~U|) zMPYEJyNWB(m6q+*geBbz0q`bE@89^*tw+B-P+eupGC4AO@&}H$4ct&`OVO3FJ7imM zxEMYxmC9T8zP8J<=k-0!BLy=$wEFKx#BX}2YwhU;R!up3Dc*7?+kQlD*zsK~0y?h~ zC%_R_HPzq1VKVA0-H6*;mheM|Assl3cjAyVD$;^|aX9Kip-VZ0;1YXj^YKDa=z=I9 zHzH}lzC%6e4>Xo5z!(;22Lg-(y*YOw%%)R(BeV;!`%|`YfK4EmA~r$1vHz~+hmU7u z=;E|9V#*V1`U`3nXGTV09g; zr4$wvqFef2m;BGBEiQbk$yZ2JrPY6=LJBw0>ezZttChloj*v1~Omk7(2qp&=m0Y7Z z9x^*rz-+H47l9rHo{^ocme8FHq>HRvuqhPMUMA>tiiL`1Ur#bvTmjs#rDAQB%4q%m zFESkVx>@BQCl4@gXRI??D7o`e`Y4@xQO0dWvG(v#@A>*g0Xj(oMG zs^mUnRw~s2Qb;_rJHKgbS9SmzH!L}KxFg>#zg}Ne_6(=^@PxTr4_B`_ z)--|S*L@T826|UN&+p^aGTF2+%bp_li;v}D2A>BX$u}fA$6(&3fO_eu2GrC2zS7jnFtpC%8K%`(h_1C zskl3(m{767?WuA(s*4<_()rvx0tOdHEvp;G91P4eMzES8-;Jy%} zA~6Gs=|NEokLysUgGhHW<3eSS2G`JOkHU@AZYO@6Vb7);sU5~qKP81CBFU2Ek8~OT z6jmtW+tT}24+cf*7ieNbO7?U={Nv`$mCH_T%v=+Ie&wy@w|8gNF4~-t-(wj4UTsu{ zwaAo*koEoB8W!i)tuEbiWHJfM-Lp59W|q}Nr()d-Jp+Bzr?4CvXJ|C!RS-1uWF6`d z7XZO7R#B#?&TjRbQf%Rcbt%qp0Ky4?SSduW>U!}*C#=vjB&T)7I?8>u?rD!j_Fi@t zcv*RNd8#cbJ~jpwhS18+WS~k_Mx}-*wXafQ?tR!oBy>+y(!hR@*)ClXi)dj2WV4^W z@r&)Re6TI$`m#yFZ~G6*XD&!G$2n{!%Z(qMfQU5e2flm9l9h7Sb;s7<)RU!es+hnQ z1_wm36CJ3M>TICqrI3}0rBOzEWp7RI9>TF=|Gg79o+aFdqCvp0m5cyR9$esySmZtu z(@+XQdmt;@(5O|!DwJsQOKE$stKDihmSxG@-gMP7}IiOTt{7 z3%V;{Pl<3=V|64>RU=}%EiBg>siZTMpfeG6MyUsbn!;WeRz!aBD=RHgwc~?NEBo{9 z1NW}n@SRUGUVV6bS7uKYi(WDKUHBcP9o035e1aW(Oa z0ccqxLc28~HH|9NyTQ}CLG!?jHBeb(^>3eD-M;x?!+rC5Zd*M1^so2*@qc!%vcL4> z%`g9pwa#t6uy$T+OGoz1f;Anz$JWgof4!mn%`JoT`i^hvJH2Be=TXpWR3~va&J*5n z7}B7Z$9d`H&U*?>oR_(#;Fb)f+|8pD3yo8w52RmvEXCeK9nQN{piyqE-19nUj924$(NG?93z^6-yi9Uo#nc~Kv%(y7+Q$S7>=u? zmS<)t$ffKpC2&03eSMtH7%!{_d>oX@uU52+A~*B^tEsl+q{M`n=>H~&O0$>Vbklbh9B53c06Om+|EGe^lk(PIeEfj?*JKwi=R9`T zH3xHnNA^_(kMjDTyngoOK04A_@;9UlkOh?UIq43b>%yU5Iu(h3tZ`q($tXJ0jlt=S zaNqls+c(s`E4Y1MxbeS_T>V&YX}KxGP+@P&>Ahvq+M6m9lXZpc7CF}($wHP&CV9_Q zukP;L^qtKOD|6DDw7SPalZMW;EWNreNK?pOh&A8D8iwVj756#s+NzUU_`Gz8>x5tI zBh|5qzA8l~s2<_x`ac{NBV=bdWM_m^cG8`DTCx*o+gtx|oo!#S;!o`Q3vhW6&dN+piHQnB<4Lp0p!b|@>PP2C>bg6YdQ07x_P_tlwY_(|vQCbbUf&wQ z;;rM)uzx!AnHgo*ZEvmFcyfN(^4w-iUhtfTjc?0Y3%~N?ZG+F;*q0^kENfl=OcIIjyLs-3q2+0*LTIGC3r+BiXq$E zH0>~E;GUHjZ;e*NA{|;>EqTd$T&^Y8CyHgzCG3U0dp`K~X#1{)nLEC>?Y1%b)wuo( zH+L_(uA%LCSmN$$-&nQdN4w>d@>4rAHe7$#!k*1#gA2YgwE4$hKPdinXUdW-hnoAh zm3L%6%gSo)_~K>xPbBlt%Wp^-SZ@U4soEVDvZtavL{uc|ii^$|hY*+CcZRMGQj?x2 zq9{0va8pIzL@(~EBB(<`D%nMAOsj;`8Lu7ccQVe4~Sy+uB1cg*lm5=v-B??Dr4%$n2Y~buu zToG6X8O)K3;rdE(%u#wKS+g(N?K!c$2@g~lt&$nVRu z>h0~zi-Y91wZRN6>$D7IND3`wiz??WGnlW^n6qOjhm@#h z-iSmu+|G>{@yg@pRa7qfggeteT}Vg!RpqOsKM%h4DWikwJcTW1fAg2L9z;vgvO*5O0*>xxSwa-H<`$~KiS zaJUD%WHO^+7v=57n#J&lMJHAtK4rgP3I%nIw66NP_jxH4h8^Y@wJ2z;=#q;Iic=`0 zz#k8%K3dg+|G~0~+Pv`NX$KJh07;<4XM^QAAAs-lyXJ*gY3M%$JTT6!Jw>_ter4MiPoUfqZ3FoT@b?oX< zUo{u~e9fq)E@*HtmbhYZ_A0Y)4IoN>djUCRXC`okbZTa- zCIO(xxhOD8-~mp4y87TuW6LFpmlB{B#>!WaB9Iznv7~|m7vt@A&iz9v`_FR^R~)MR z5*|5$i{?t5qc7|jePD5JbwsJ9G_EqSbY;!*8yrdIAvtWsk~m{zWvwb?^uFFXYw~iM z!VMR~taGmG1FMXX1+V@6YiP~Dajd5ZymFSNUq}0fPML z43tT@TQK= zl`Xc|L^h;VHO{JUvHZY*1pGYGjoh>{Qy6iW(_`Q(0L36i#KkR=fn)d()I8Rg9LZDx z(T8%Vh6{fN;B(5M#VF4zB!ohp+1WF*XT}jJQH<~ONr~a$OifObYJBE&NlIQhJWPA! z;Su{C=2QcABnS0jwJ?sfoF^WE>bQ#XOtpyw?Ixg$E(xe*f(>IR%!8lUHq6vPVGLdm zBa}GsvuJec zoRXgE#WNmzEL>lQZC7|hS#bHv_SM^ZvtqMN+L4H5^S~<1^-lf)7yxzKSnM;f7h*KP`O|%TL4@S&M#3@+o#{2p>r-;yBus2=Xx8{yD z;i7g5Pp0ik0H zRg~-Jq-t88!YWf-&*A|*38O7KaATDBlZYsQ+6F`@vwA)0oKC5SY$Tx&1>7j2(&zf_ z#!WGjjV9m+@-*BVN0GS^ZXA)cbm57l=tNwFnBBPAILI=+u=Unpzc;>K4JvYd+7p?hjs1@$Pb!vr0=+l44^b!h~8@tK}80`S~p=rSgtM@WiaOBxhV#39hzU zBxrwJ<{)#E3sw494C=V++K$rE?xZL~WMc2q?c0ozrUgNvr5%koZ`}FZNY%<&$x)`n z8KVoDw>QMZEYOAM%i0=miq2oWySaPAg1nHq^BXOh-JLC4P|M*Bsc2Syf93pZ<}4U# zv4ypEw}cfB&uSW~(o3(4Vo60Fy38>l=hFMCkEL%4g~EdlBXZ$|tQaVYbXu%9Ep}jw zeD9mdNRJ}|&3*-m+I3_wkOug#C_^~2I^4X<#JCuZ1^7C*6=v5gynZm7+-Kl zM~~T*gt({(w3#*tg{+Xf1PXY!9{AS^kq4lNaW2PQ>U1F<0L_&u78>zurH6L4e`8PE z;l#8kV{&d~epY{T)4QWry*ImKTg|MD^6>RROWZFd;qSram%oVZIQ zPUBf66A?{<5-)JGyX)5gcPL;3TJt<6;-vX~isQnbiZ;}sv|x)-FU({!b!Irw6lH7z zE=Z+uqQZp>mrgeQ|j9CZ*S{3GqYsg{MbliXmrTB>YjZi z^ZV7c<@3{XHdUk*MW;k3Mn%u9$(vbInm#KrJuxq>;!sX}O-V9B+Jt47KGkd&Z$_>_ zC%R~_6qY&`ujuP&C@;#eMd~%iVDvE3z~O)>HxyP?n@yroqc&3g3zR?XwHT0xA*lFI zoITQqFeP-hWlI*$Z<&ip8y(eXk&|u>OHH)~A#}iw%KFO6JpYAz6;4>UI>!+)@dKg+ z&l7k+;#WyjRw%JMe}kLcUSvz5k2KrE*4aouOKw?e={ztqKB29& zbZ%;@ZQ;z;rkLoi!n}_3Z(0}VITwAz#K$I?zjMjgs8 zXI(WJ88xldj=AMCb1LlhiH_P08|o_1gA6Ap2i*5{>7-y1qJ=>hXM=|X>4k{0MY$ZU zun(2Tfp`XG4I`H0sEL$y>###hnc_KqwF@?z&C%v)#Q0flAxU~igO>ni2P4ziDXTiL-%@y+qj-Zt@wyqUdKIrg!1lAho) zTDbI%dW-r`!b0J0M^Irtl8OrQk`j@K!=-cr@g0B(n;BvvoN5#}4}!PU>J^GX7}I#f zw9Wmnn$%F3uUv!3nCpBLJl!>AfTt8eZCTLO1j0k}q|8(;t_}7`B}E5U2J+r1H-*Ru z6@sn6v9;;+EeSd>^mGytw~Qwt!4U58NR3SDS+a?TC%2Y-0)($MU6WfjZIZ5nD^zF?Z)d`h(s8z1lzi`W`OBFXI*5{VZElVjX$xt`5^k%?-lm&-`q!tzkAy!+k zDA*8hYLz~TeJXbBg9CqJ70-Oiw!Qhz9IjglgO3${TB7|&$3x98)vJZl`oEe zMV`+Nh_hIQ{L1+EW@n{_IVI%+woRS7%oq{-94)ITy-C7+rd~(OBuez{3eg&X?n2?s4mYI>BVo!k|GCnRkl0wH3B%l?NS+a!( zT9CF85}Ax{B!yNenIs!&Q*h_CrRR$2be8c?{I6}V?zm}))m0l4R*$7Kp>}lky)6F8 zh4-!)j8Qc#d1e)>Sfw`g{PLCW%g^twdPF{Ws`EcS*!&;NdfkjP`Mb{~|v>A z>=CIOqyg$_#a=Cd4wedA^lpbC2x>HR1o!0LoTO!-*G&9y!j){`Y7OOH+Pczj$w=?lBEdAX( z<8xvVzq;jThb$>^FN|+Lg|!#U<*4p>LI^_P?-U2_$o|#nG-!w_Q;c^^EoT}?wkCe# z22p&v@y7M*PRyBeV%_>18;>Wp&zaMal+-b2PJ3ef))%*Id2#Eu7q7bN#cfL-8XkUV zN&kaGLl0sL5Uc5g1F^C<@jWqI`lonYYUOa%C@$Sl4{%(&=dsnWtp@UG>t|Z7*!u^1`gWF@nA^l=@eTFYm{xa-P%kbJ^p6YT z)8c~j#V965bX5%2^t7gXTDUb&1I)m`Qyzn?wJ%=bUQ z40qls|MP(Z?;JYx&Vl{EzW(}O(>-~W+$45!+MMA?H|h|wMEo91N<0X=8lLFI4_=%M z)(u;DaxhUS>J*hi3(ywZp$MRux!5IFT_5*4OU@59?UkEKRq^8C$Sl5f0upE!_cJYy}&oBz*%mEC(g2AHqVrYal zSOwZc(ijStIl+mHWS`1ahvNyF#J#0zDN;O=1XwH)DmAC41!9HzPSDd=T=c{|^aOms z#@NN_X}b6zs=6FLq~J1DjCeG;xNAJv5X|wRyaV_^Z(A${GFO@nS_u~BIP@kxf*$>1 zgdDvAFoLhJJ-rug5!fIXMt~BvvO{QbGkbi_>!C67TkL^`n9$ezbu1=CDro6{a4aGu zh9PK@Ca3`|JO)}w7DgQgGhl#7=Nvj3GR;}~N=;cLoHDT}X;d@Uu?xQ$oDTt9U@>1YrP_Ak>@|pVzYCnav08X+=TBAF&To zS06dg#OE6Fmpkmiuj_*=t6KLqFFMkEt$gmSJL!y*&fKrw4j=lnRC`E^i#hl&*e_-2 zqY|iwd|LYA(wX&&&55+4bg%q4qOfr@oRU z{?=Am)qSY#=NGyko(J45hfGg~job#_VMK zdSvsXgq04iMVF1;=_&wt*4yo2 zHob{wlsM%EU1uT*1w8a7MU!7#P@wz;3du*?9B=WWO%TP$mY!XH_*hzoE zKS}>7o*Een>d+?3c@5_p!IfgZ3Y3E|u~d6%syz+pqmtPQohT+WXAYTyKW%Y=6PXj3 zl45Zs^sQffX8H98(=rTk+RT{3r2cgkz1c-kgukm+HDpv-r1k>VC5C3_tjR@@0Vq+YyFp2W2Tk5UF^CCz zoV}iMLcJ)-X^r(wM=W-m#wJh`;lPOm>8YFke02AteU(+F9CKlGeOdR>u9e4YqHCGH0vx_e3W2pX^#SnyV47 zW~_2Y-+}99M1D&ZkyqEgdrr^U^;^zmshUo>`8oN8j$X{k7E#0yWJCP981_RlM1zr` zAhr$+4G9aVeX$=#C(2EvGWFaB265;Nm>JT7cnSlK1IL0OCC`~M-Z^TWmd+utm!@D3 zDO`aa;>6R#Czt{zv3PQJEWhRNr;ub@Q4Q=>W%TARySa1VM14jBVXynP{?^@%6{{5V z?fJ>>ZQtF<(I>qH{B>;N_`AsQw{nw$ztSyz-8-t&LSIvd1A*7J_k4BTRgVIJWI_NN zcG)%zDAs^ZR3=nag4-B2R&ojp>=4pF_zi)6LLidDC3F~#Mzhgu4MAx|ttmE34dXH- zIV-dl+EAL|>CwINFneOz=wIZYy&}s@>}Ikou$40P=kVAtzQNybxb$=JyHXAAlxAV! zMd$04V!aXcPP`C;nu<^I{CM7yEEeD~sb})n617#gcteM}w^%&63lJwJZGcfFE>!_N9#E45 zK|NG=!0BSAlqqjBKL7`DDUNop=jlwk9>s)HJf)cV5v@$A1(eqp8e%cQv>MOiDQN@I z$`eE(f^QTG4Yxs1GEl->5B#t$Hz-!yXl+d?A8i@GS^V0G7hgn@+0E>=i;>^j`EsRNg+s1 znt-}Qmgr7?B=P5#lx|9+)T+RwRyy)k`3d%#oaIey`xIg;Yr5^~HgP>rLi7flY~?t~ z7m9@MJ4{7{mgFSF#*v~#xXE!5n-__#Yy;&@TsF~;2S=#D>qTy^kpTzcC6|BRi8JR* zfHRO>VL|o`d#W$qh~$)#E&=gozXF;ACRv*g(FI0pz7OG*5|&B7uYkUPo*jUXquf!F z2){n5_Q-y8>7FVCjH=WKn<8_6>j>tI!h1U$s+?y$i@>12hx}*1=+jm!1ZBmL&wZ&|$?9E!c zD&E4*4gR+ne=U~>?xsZlg`UC{q#AlH_iDb}e48;+Dh#fU zZX76buQz^m-qCxRao?G&+QgJpEI6dHH=8f`k+pv^gdKFR^8{XPox3;*fLMz@MnY!k-rY(20MI;4@ z-+NfzrjCjDJ)M~>$U+-zDh13yx#=2J6r>$-C34~-4^sn+)Mf$I+q68Z7VWp-!Bx7- zkb0oiuJkZkMG#Uo79@<)KrIyZD;Pjl@DL-qX@wT&WJ!~y(PpdTMH6U zP5>^O_l6%Ic9dFYXl6%`*w)|HvFSU1+7TYHPI_`vj2LQ>w?%|Ee)Hm2mR=`cO0L^j zTTrJHpN+Jyy}fDWiT)h+nhE_u7NKBlaAcxM{$tQh z?G6fFA^xa8L>2M7py1lGH!T@{E-H87tb(~(@tI)#hT-Kq%2=*GOw{y$xG63|e=eo# z#JW{yU_}KpbKQNy!8x)WnZaC_2hKzuB&kM#0l2)=POw$go}QYD4m#xRP`hn%0o`|;7S(py@5HCOPxkfQypTnW=Un|tSn0;{eYYK6-IFztp<;c@mMuBkcd&WxJ%@h9 z^etykRAefGt@%8Bc6E`t0K-n)~Y_QiHbmDP8srRko-*q&BVb1ydN=DmHZTpI-J zPPmhOpYPEY@WKMz3nGPd#L&SzC5cwh6mdf`4hGyP?nj>1M%hzCB5YKn%(+G-4kA1~ zMNmjE!ixq&U~ec<6F#Rw z^ILzrZ#N71uYa)7pX^u5AO54cbmI%Ro_@G&W>R_NPvuWnlKB&K)QA|cTci21*gKU#e5 zUw4I_dUkC_<^DA*w-s$Uq&AvVul}C{zlvt9to9fS`{>E6k(zJ+`cLbw`|GV+zqRJT z3%S~yd{f^4eD&799sxY*RI@B(09o8>Th=aF8UZVzxWL6_-xi_&7-gWV%_M^uHSO5=fKxYh7(NmXyEA038 zHPSV$))4NnXW=#m<9~6ocu{V?fnAi>gf+rCRxzHQDgM`{w=o*@=!64$#gFj;`RSQ> zM4AJ*;1BkOi~7HWD{qEu##0h9#}tO4kg9LSrR&=%;B^8_;Upe_kI1bzqMqQI@YzsO zB?9<2qi;F+z(VoI>NDU2KXLJa33^>A^ty}C>n7oq;E#2||Ap~Nj(-oB-gwoBDJzQ7(hbQIF%$|Yxq-dA) zf_cO%pcekP1kn{zi$K~ERH{dKuHuRkFD|%g@u|nQHkYpc+SR+hRx~#yYjO2m%j@cU zuUb2!JmUI&`AusEx>vk4dwunFgA13(g!MNK@7&s3)Ls%~Dd=MDt({$^j&57rJo$8U zOKx;WUHghYr;NiIfY+Y^uM+pK9EX2JSyPKc?_QHal~)?v^pvt>J)%F|&XH*^`S`+{ z`%gZ$b#Cd3yRP1KZ*fb?j3th*Ew7us;HtG5Wzom>6gI3J=wAJs*&|ig4lZ049ns%3 zvU6K+QAbIzxnMq<*V5im=2*y=+|-&I)mhiRYBAALo7f21NAW8^chS7$iC@9Ka$4&~ zM6g}Rr+v+|Md&ZPuX(xIGg4G$#Ax!%*cEN-#UoSis`R&^!v}{JEQ=0XI(K;cRkTFD ztDP+!oz5lR+T2FF+OZNPHk@+tqNW@BN(V?1Hb5@MgHI>IYh5m2U-L5KOnglN_H~ND zs;5eebCAZ4ef5ZX$Qp`;^mWr?>l~D|H$!*+(Y9J%pd74m~mjkb>C>-;iX?jU@xJ^p#Kj!XFU1~f>q#B zfb>K{0SQv#+4JCsN=*(WEt3;%ab!eEskg_|DdL?z;*t{a>3ttQJahlcT){luLn`Nj zBR$KOEnU4RYgvZGl5hFdu2xZf@5$_xlj|2BoIB_4(S2w3_Ag$O8W)y??Qxz%#5ZEU zeu)o6oI^#|B)EH@RKRwrFq5F?aJzKPN#9Of8+1+Q@?0v+)XO`5`Q@=+-}26`c2RxL zsqB=S*Y_R5`Znx4v#-BzO-gK-Oj_#Z_csE2Tm14;3I)=2iEj9LlFQOI4L z7ZWWAw&duHm<)`LO0$RP=rk2aLP6uiMt3r@>4b!bp+FWKSERoo@|H^kW|)2LuWk)r zcz5^qXV+c(tAhvMy82u49SwEIpiy3P3>xKG`JHPu>RrFRXKvYs=Xc%q+pBlWAHMgA zeBvkK!SdXsSrI>Fx|I*CJ^e6%>x2&10^X1z%ziSIpob`>dwdAKLvY4w4l^{SLAriM z;qbsv1&cYtHu#|t2k0pjwm=+f1{G9Lr=tt4Gj6~caa)Rnc9)sTy(YFN?3H)>Dz9F< zVtbK&%|KPN%4AdzFWr29@2=mSJ^B8@Z^_R^k1Twj#ofE<&FeRRy=TsmMPa0TW^eh{ z@ZsMdJ0}1A#n~ZLm-y0fH+f=l#lCGfo9 z6RfzcY5*VCj5&b24&_!6d_~K$Hs9HK;N7F|GRxv~pUTjVFaG@htULJYL(q<|K5roa zLqj$_J9^t+j>vDy-@QTp_>ZV0Kro^op!f}*$DHA?hj2s(LMD8{87e48p-|VoFbPP( zRVTz=Xf$Ky{fF|?=hr*hQaTnTq^nG-+KP==&A+Md#QDJ&*{6=eyPoVu=tbkks@7q> zw!j#1%j&)xyPkYu!}(#M*!>U$AWQ7yC!OWDbm^J%pG>CB==Y>LGdVWDb` zo$#TNY_qD?-j?&Jc)ZqM#EM0$9u`ouNTaf7D0)$eLNS{S1EwJKW`pmSY(OsChVtIo zUu+`1-syzEn#fS1(D1wApyfRQ^xF{@zm0`R3KAe>h;q9l;@feP1;!%`W9)T2Vuv$qRq!xe-IZXnDIPE?a zm5iVdC5H4!=R@vq$-)_UEYWk|vBVo=cs4KCy`R|weyNmy$6kjB$l$S{zMUFJLXc3O zC6eDcCqu*uI8Vy|R= zLiLr*Psm3rKcrGyP`7e=Fcn`M? z7v|+=q$MRpMNHW6pa2{0Hk&C*wC_3fp%2yFW1k#Ay}1N0`dA(COv4*Qi*aV7Qc|#)Gm#sXM#*6zO5nX2>6}zR3i%VB0BrBO z0>1|#%$Bp^iN{-WpC<}kYl}?G_YOmse>UmtcMU#rXIF1;r0THN>T%odX`8ZU6RO(+h2o#zszdea={V4m2`euB96IWKu%}vt z1aO`6yn2xpF-NEZkQ5{8Bq|p|F~GF%g?f{yku=~eP`lL(3<6zzd#JKGwMZL;)S{p^ z2wzuF8#e@(3VMwm9O6nxqQrS(qS2U`oS2*t7mePV!9hmonrOR(;5-Xr%85aEg}_Zh zGQw6E`edJVhsC`=w)I-+_;{;}VIw6>s`n0+Q}s?d@&=R;mLK6Ubl`_{?!OLvkAOU2f0FEqMimOL9aFGhg68qC$);o z?1Kq=X$3{}m4O|catfcZbh^LjduemO=zHlbS8}nA{u!5nd(gFXQWk09ycN?r%9*{= z;W)XT;fiLEBnYL&1^Icf1ly5c92q`AFR=L63ly`W(@7gJ#>0=gs2Z3e?8Co4Denjz z459*U{g38=leRL1QntZiQnXAPGw8|FMwFF}`xL2|!!8wbEo}7$quwy2LPk7Ml+ox* zQQ7nrbh>~;eZ~+aUFmy>g0A#EL^)`j44u=&bxvRS23<>=0N)i3abFG+1firTCoA2a zjGzV|z0n+yKInD`0=bOO*dKWmU*3%zfI5y}&bh4UhIcyYBaZD+c#ocig72V8th*sq z8t4NZG|{;R5g&T3!g_F%TdP9(Fc8Q@Zj(lz>OmU;26>()s+j)VJU4IJa})KXdp(Ui z6BDi&OCsCQ|0mW={K z8tj`tI0m7xLNpNI5E48rXR5~`3OtW1q97_ZwxO;dFDE-QBL(HzV-xVd88#zTN(tyI zo1{BLMylPVR3>^1H4pf4pULm|s1}KPrDvw=Qd8}iq;()8Jk`t3HmA+yXA=}TagUP^ zpf;~dPUu-2(-}Q-0%AeZ8x%QVB+bK!PPRs!TTW=Te&qxmr$9py)8;)!krRO)cFCiO zW88A$3f?n8PWTx&=vvwY_&joAT0>k1-vK#MRFIpUZcj>}raBYOegHYagQli*llT)9 z-IsR(`DYbRUf!KTe#vU*{ZxV-VL7|#h@@zpHl{JRYeZ2~%S@t!>5X(fB}Ux|gF=v` z3POxvkeQmEL9I0f!=|e@l4`LZr6SClAoDP6y-Y+R%P>?qquvB#F4aI$TOn6qayA&0 zBnQP#NEf#(pf(?J{k_f8H5 z<(V7|%8?-mOP4I_?&@rrJG;K7s;ndr-9o83M_dfGk#L$WLp)Mu3Im9nr%kqCU;a|+ zqQ`Gt##|zO=+}ZB=$7iD*RH{b@A79-8UOb5Gb|wi6gsB+7TLpE*m*~Yt1dz#Gg@jv z!qkL|dNtI-S*FxSaMG*^;?S&8o4L`+q7nvi!!_#30RiLkK#&$PlGCgPi8)H?o|f}e z*XxT+1Yv71cuz#cU*iiAy^{wd4F!g6;xt6+%|R4j3|sSl-}!(Z3U9j1Y*?2!^~HY4 z$$Uw<*O8Et(%dw=p}tO`;~5!VCcI#n@L*V-f_fSEqD&{!{u`GeQ}I=W8?!#45D1A98z1UqB5@Z!Mg4v>hvkZJy;(Qeykb3YIoywpa3PBI*$#xZ` zxo1BN96{zF4S4GmZy^q73A)mEka+lCT7ct!6?S5^=|Mq8WAogGy6Vc(Vrt4v%|9Z; zgRDVTk0mM8za=RYBIXOTR?QQql~l~r@*g>6o{*oH8?T6!dh$0FqtxdQk_}1>YS~mh z%@lRwpKaV;^$>AV)DL>a5t*HZEWJcDJxyRi>SfGg?E}?%C>1r2Q0xMRy`bPADR{qN zLY+txIe7(jkOqCPgM?sJP%!aS(W-(c!75lxRtucGS~j2$ap?hQn`Sl(15}C5<23At zYhJHKsXMrezwlf115o!lyPfn7(nfo~=+uCXY$sanWnQ2hg)a{KWigZj=>NAdL{h15 z>RrIS8@5aI&Y5D*4Ted3K=BPr|IKgU`kC2m9Do&QozWG5FfjQoXu6^?!1?4$yVG5d z?~5G&EAN01^(=;8Scy~35QLQjOPBO7UerCWV{TJ@ZDmDCQC@Clx-B_AHX_VoMxjLV zXotGg!Y|C4rfKuMQ?X&(dPO`-8lOV`D*EC0#c4CK@!gjb$ZF2t9)Ub2{??0>mQ7Nw zBf^WvnbbooyXV5sPI@8{cl%*k6DK5=^>14-v8aFdiissb!wrN*ucPaCk2njO zxMX5M?O$?6&Iv3QAC`moBwsk)EXTR)Q^D&yF~vN6`8mN-JixkHN~$1?Zdlsi+1^kW zfSCmUJD7V=SNfvAInSS-*1j(lL!7caQN+CD^$>bwBux1-ehE znCb|!LOVx}sL_&x>!`~CSm83AAWjw6Qj>~%{>&N^?xW6oC>rEwgk=(RB(Be}0Ua?L zs#&ZAS(OK#0)sHxA|P+mb*VKNCQLw_jHg5?qeC%Ebb8?}Jnp6_qtM=yj;qK?2)~GnNeh zu;i>F6ONK7sL(O3Td$nTCiNgiXwu;!r*JfNIw~^ayV|PsKwnKjXw%h`9e9&=6sLS!ll01WubX zd@Vv-3*A!75ZN{}huSnIqo+IReL7^ugN(db^)zfH&Wr2x{Fcv5K~d6Ar{l~Ve=8u3 zjaOaHuc-$8k*`akzXtSa=x0wltS$;%&|X^vPlAgG%Ur}0kYu2?Ez>xID@-O|;wy2D zr$?qDz6qm==)TO`i5q_Lw-c$(x&qj_qG2*|U2~ltM(ryb41VVs%ry!}k{~SZoja!j z%^v*Fr#rx6dW_9|dzrS{a8OYj&P_$E|GW}KAu!F3ja`vqAdt^?r<;z=$oVJ|9PEO_ zifUeDm)4aP*rQZ>HyzJ}-vWJ=QIS)pM`$D*FNlytR0mI>RV$8q?$~sBuoayEWojl2 zqbDI?nY7kD0eM3`gNSOsz`N+)aK-N;3eQylgpJ3!uqh!V$}X73C{EFOy(egBdV@g8 zo25g`jK| zs-Ng|>H$<@;F%}`Z9!&{+`z#G(b=1=9FM6{Hw+PSYC#8WW60lJ2nSd-VJt!dpm+rb zZ`wGtcE$4kzU~E`?Q`eU*32p+mu*H`GU}lQhX#j+rP)HMwTHrITp3>x5}TZ}!CY>B zB=S%A-Dj*j901R`yYkTQKF?`{*+++mjyEL=FW8*RUOgz^# ze||^D{P|5|Pt8?+o2x(WJ!k08^uHVLg!nSbUXpq~yGOlL z-opRb#s52AclYzp-+lKBFNnXt`@7%08~@}bUL$rnzrEY_)9X7<7;Dg%Y&rIx@|9-_ z*FKdIjcw)GA#`3dsRSHR@KYRnYlVs&&*yKHdE+_8$2U zuq%4Tcs+?so%Hm$SV2ginT{Ynv@A=DOT+tP>?sixyTP0-cD>WIlx)~=$yE>!LE)05 z^v#sm9Rs)Bdu&0BPU%ecy{&_Hc9a&)om*7g+%lS%S6-fGXPa1N(qp%uIQCPK_aW=M zfApG1*V%q6Z)+(kZfY(rn#+!gN2}+|sUF`(If<9vXVl|aCxi)eDLWpXMCitZ5VWNR zpBI1-?g>}AQj6=Tol!V^Jah&n;UJhhM-e{ny@)svT4dN?>uZ zXw;pIbYO|n_uZI4>23*E1}fN36jA0ULHq+%;IK`lgii^*7e zcgJG+vo+PF_r=!uAgp9glN?{v^H<6AhTr z?P3o1eUkG$G~u6ZmkB_)sk-PB(3XnjWNa7d=cJpNlT zzH{uOgxuVOq^vCY*NH_Zz8CUrMpC!R1~lG^@&{#3II|oXPU7=M=)`vD2q&%lhXkhlmvV=3>at~`BMZb$w zN3xORWh*{rJi(ZuL@Yk&z=Od;aJqz>eQ28KT{=?KerQgJ4xa6(Z3>P~ut?G#X8qe; zdg37ZyZ$DBxJP*h?&tK55Fy?XgUDCq9l}A(&bHx+5FsRk#4;wII6I~{A|S~?^BY1U zqKsn2UKY&6H?P&wE40_XDa-QT_R=~|;vL7azF?v9No5%dqau}n;WCb%%G4aoI%#o= zX5e05uB0HC$~|h|HN`|&Djc=dmdF^BSh4OeKRc?d?dZ?`vJO+yd)XHL-Vmyzt&NgLP+rIP1Pe!q4d%^2&7CysX0c_=O zRR#V|7p&qVnA!a~dyn0${;Q-#HCKUsru5}upSk<;Y!HUnHPU9#TBM-#JQBdW5zFLr zgov(@cI{1P)LYI)?CjL{7KRBEuH?BbrwMK~l zh;URXZDK>}bEtQf#G?noJ)1Cquh~3%nrMvT*$vDhf^&oy^2$QWn@dXBP+CW^c)$=| zU)Hg>dtIXXtbNJm20dDwXU7!HUERCtj=@NJPp9|_dyBsZv}{6-STjCEqZ*=T3JgU> z2G|iTLO2VT^vsJgUtXzDJ^pGibf%wXGTUOk=wNHsO);o}AvE)a^$=k=7Z&MW7%H?Nr`EHw@vfWKch&7w z$283(MxM zQkPwKZ*)uYzy?)bW@+s@!V&gjFK`qp`{wzoBLUH6i<3P-r%fVf&Z5FEAU%&OkiO~ui<#xU!mGeol)c;7<2 zPooe11t!HFy@)+}7_<=Qhz@6LE=QTy3-LNymggMeZDI7fNOo1iz>4~vt6CPsnjVs_ zSuDO7*|YcF6s~;BaQW&`%^f*^{j0yj>|D>LygPs0L&GD*#elu05dqsss=|=EUsO`T!PKjLCeVu z?78+C#PBGsif&GmHbYH9K&hPI*&)e7J5;?v1%}j6+Ob&fD{O6!YE9`Vr#)*d?d);x zS@j*ujGf`zTTm7Q{fL{ z(8wtRgQgN=;xf`*A&1?G-fGBV4mLC}mJGGy!kQb`us->LH8)Cc%g?YzGzx5_`5Vyl z-mYHG=SLTyqr!jzDM$PKia0bw#E*q|JOAobvf#2cpx9Xg%Ps52s85#0o+ z33)PA4z?yUbg7jp#z(|k<#(BF{I}Q0!&1{BSkC!`zsiUzw;gJN}`*TA}#f_&Sy%|5`k?g><6_s4ZjQN`QeDnd~?=$gR>PG;dLrIIZV)GfWsO*U%31DmD zpZQ=GEk7}bT2NbY`G+{-^zbVCNPdBRCY|peJBRr{!~B2d^Oreu|LLtXJ)u+NiV4YT zd<0Ix@k0A1LexA-DiA( zj5V8g0eqq%lT?JYnuYtG(88KGm#_EAzUC>O2M#6^}KG&vE)x)~L{@``sW={%-ePy1#L)QTfK-8kKMS ztx@^L-x}$g%dfG>Af}f_#9)o0E`0ulvwV$X&8nF9*jwy%cH!B^N9ApDjjWa5|NIHQ zMo~v={J)oG#pk?ewo^9owYvC?LbERZ;p8`NigkVNn z4e0{#i$K|q5k;NCZ_ZwrAFggx?fbk_ejmG5BX4`O@j3Q5_VO)-e}X>Op#VdM!f(P! zC_+BCm^q{b6FU@M>abu0%&;OE=oKQPS|S055G$HIgt$?_A;g&`4{3ru0Dt@cRU*W3I0bA$xIrdP)vXkL@|>_CTnPzojdOOahyd!&(UwJoih_}4+ z4Dpe76Nh-qJI@dwc{g#0x4iQV@sW2Ehj`07*AQ=c=k>G~B9rY_y16sD#Ds*D6!IP( zNcq0-MC!(Op6=`1bGkiN{N=w!HTU5v^BA+8+Ocg+-b6Z~C=@^^ykN+Hy(W>}qDYAZ z7~Z6VXfGp(hZNI8FE>35pcOLQ2Tqhn@Tc$vHAsdh1<#9T-1Iy39JB|4=b*iqdJfu@ zz;n>%Og#tfQs6mgx2B$hwk^;cQw{g*BRS^cVxl8Mtk{j9WV@E`Vz`o_yr``hPKOYf zlq6Cdk<8COL1kbjM>U_?)7SU(I~!A}no1`^4C`9jE2DkoJVxOwzGH||1CLP{%fvB(9`~Y1 zvRiRlR4KF=g4QR*UErj~z4JnB`{_NMoc8oD|H}l*+5~z##cW_Q(OMlxALlQekC%Jn z92XY=L+p(F(jKl7M4uBoA*6fQiLx+%g9zz<26?dVZxCVK&ma%l{S6|t`x)fHyT3t% z_eq1Ch@bTQWe`7s2{74%__1cqP0oXUN`4m+i0}UkmOD|ApH9Ah!pYY)2uf9h+Hr4$ zCx|MJlP60AU-uHF5z15ehCFHv;X2+THDSs*IA0Gq2j}Zk&cXS5z&SWypK=b)*8|SM z`TCS|aK0X34#yOz@Y+mEaBy&9aAI;M1U2^v_y}rPV0{jj7bo}dcX`+N8{#Z%_Twy_ z__w^taO;@tJawm8IzEq`>fb8 z+@=D5BWQakji6oScX)pzXfr2`pgjyYg0^nb2->lLBWR<1Mg(}+vn@}$)h1t&>a8}t zq50&~Pl4ku`CU?*NM;H@lXKWVH2K1AA=8momYo!aQe4zz8qrdy%`xO;Y62RQqDV%q zHI$70j_LkXzmPk7c5c?}hEKog{6&4e^A~xo^Ov#DoWHPI=Py|GV{#LVL_DY@ zz&1z$`DE%}L9U0@FhJ5@if00U^^%RB$dPRPkwZgrllBak(~m(4N2-61IntCka`&hT zn#1F{;4>hoBH1|u#*hHtBkG0cHDAGaE4SDwt`Nnc5Q;~iZ)gzpD)KRl1g{v&;nh;q zIzh~1Dps^n2nC0k{sz#_zj+_7U>=u6Z^b_`7O;;g@PE3RuL1L7@>sqGt&pv(!fPJB z4x?w@ZO&N{0L$0nHLLpfpV2%7E7rw^_`0-0xFf`CIC)@sv?T^g1lCYP!^6rdKOe

*ISXQgFWhA5#^yI`n=X3I^fZxcdGG^b|#zsNseTH`dLGR>bjbm`>U(v8Q33@ z1&#kr3f3&;`{QSCDGZSI75}8*y?f=|d-ja~O}F8HZqaQ3X-qwyB5^M>UJ{D%{{MEq z-$*fT*jAKxVyywFjL*g*nF|{EVQE#@*qtt*K66N`yT<4^V?{V)QlgvoqS2oL!NfRG zD)`fdZC>#MP6+ub6P<_3>tHE|OB7HbPx`s^J@C*F0kH`|denNye67gymAqsAlvsk$ zcV%a`^fYW2#-Leg-k6 zX7)E=4?l(A@nWVG%+ed<#X`vAYW36O#R#7J|G0Y-_^7J0fBc@iP40s^A41+i*zM?^)Xh)PjgH?-2Kb+1*lYFpb{m$sJI zSINws|Mxlf&df~+BDMYfKW_uc+`0GMbDr~@=RD^*&v}-ILyQZ;v6vuiRsIMYPkkLC ziw&?ZvMPl*KxU=*khpX$i7om9!!0(<=A@=;7_|7!OyzAp^HS|@-<5dOcO@tLfT&U;bda zmdDB@AVs#t4oZwl3=NhemB^?;LA16*p=$1kxr55Y z-ZJ@R&t3!SDx(+dx9iv*A3&Z}J|NNXUZFwllD7jMTEilR^4gLJc+nz!hdj<8ci8B0 z#}2GxaO{|l)l8mCjvcVr!A~D6C9)1-7!QC zSku*3`7%g)y45zfxk7_5xdJ8(iz{bZRw^yUX=&+Xt%i)?IQa9W}mwRsBeq!m;6Whhb3+{Po$vS#aTHuWj1&+8Q(mp3d7-wxWeZ%2tR^%0Y~- z;h7>X;`4_aQ|mf>w4A1Q5lJiXZ{YTO+vNQN!*JnkXFvfMyff5RB^`ZE6fk>{8H$X! z|An3_gzB^O-kmPlJKZPyHUF`4efjBSeee#?6B=MLL^JiCn7=TI36aPLC5wU$!Xr#C z2nl*>T&JBblVQn@Fg%H#J>p^=?Vg*(<;GxKxQ6h4*@X-4d3ni_m+o0`2L3^7!~!IQ zHx}S|)*l2CJ0~f!6?p-IR2U5=lD!=SQ#9-t5DvAK55zAAhUT_`;W3RxVQpRllNx^@ zHfT`Jf)USLJ|_JES)~sO$4|btl-+2o>uuu?C+NFkZo@|+KNcJ^7hA{`@g!JB6SR^@ z$R&F*HlL*V!8DKqS=Y|&D``VdQOa&yf*#&rS-rJSj{Y*`DP)?R5DqI>O{XR5OjCZeqj1m`Dwr&1zmoD9&((1fx z?PcFy1X)7x)=8f$utS8LM0SMuuIyLZVP}i);Gxm%7~#QP=$+whm9GFE)JGuhmE=lT zr4D%X~5oc5e|iel|s?m>^E5#|2+?Qe5c=b7cOxuE)7>}!vsNqkp( zPR+{x_B|>GyE|>3R3x25eHNkmD8v-ZZ1^)$+!ySXu&PV|3XMG&uDGfYcj3HP8!F4R z=ZL{Zj5d&xi{fXCPp$i9F5B+8ahh8>i?WvYeHOb@EEk6Y7lY3I9kfxx z4#<nq3GY7Tafm0`3tiN1?-KR`$nSf-OF}~L z68R?mRY7#$ColF+u#ClOm?Dg>t&1NV8zWm9LqjYO6vx4s)IurNAgwP7RiMa*aVveo zZWb;>_&y1l9}v^u3R%R8KZC;5lw8Sf!7+v<)5U+wOIFw zceGW^xaGl$ho7JBnsVo~s>wh8%N-SWsF}M<+=D)MM}1bZyY$OJzr+91-RPsxD89u$ z!CY5_k-Fnu6mF7XG)$stZp|a!YvcFu6?do8UBkS&pOG*lE9@xtiT3`bO8Xkmm*RD( zhUTxojVNK3-bw;OO-Qgrmf@T&$`aQ13!HWr+ghuu-;Pdh0gIE}z>Q{GA%vE!z7ZB<1ENrDSD^Z;96dD0fa$N_KWi zQVzieKOew$M7bKZKnH{&;2T3>wUt3V!E6@l&%D-utWmC>{ReG3Bkw)EKYzzNq44j6 zHWhj3z|o_^@c@j;FjD89Q5}T0me`DRZtAT2d-EMrlwNZO2P`Ei7cIy(9jqL--QN^m z?*x3WoCQ9si;mAbTbXT*k~>@b2H}@3A^2vYt1JLt6o6n&Tmr^JGXF-$O~+UUd}eRp zOkVS+SpbbdY@sG(TT%|k-T(NTLQNQ3DH!wv&g`Qf!U2{CZuhSbD5bJD@S+> zj!4!f6`5|pAC=Z9J`Lj0-q*<|TE{i7KNRcdM7@Q0i1?(`;nYF#@*mNhsOt{Rz6Wb%Y!ogDno6Ykh-i$lN% zN)M`gu5#Z$Q*Xtp8IcjWg-++r*-b4;MFlg|u!<~)WBdFG%}GUDeHf5$>z0P+INKMM zw z0=_TI@Q&I z+vVbWbVENvVE<}HK~Yjm)BNoYM^P51LL_be7 zcs@i~;ff=^=DA@7yk}M^?Vg*&C7Qx^5`L1UIPqf8xC(g>Ml!pZxElbJUM-u-dYn41>`R8xiN;t6RFW(9%?J%$BQ5KEx^ zWY7h(*s0>ho*m-F?CSk&$=#ks^q1-vvr-|^x|(*>rIH&PF!NNscamQr8imwSWBfkQG}m6Jh2`l(0u_p zV68v~iipNRE1&tuLhl#V&+Jy-Cs7QwXiu&EDx$&R%2#ruhJJA^C5iz;dcK;YVYuUOk^+hetbZtQchMCbqj1lDYiv&R@-<$eGN!2<%r z7M}&O)Buo#Q&6P57;q6zZ9Wg|0h;$UZg_b5F>D@TX=!4N_RDT&jm^uh$kHw}kGBc; zsew`8<1DV1mP)7KbN?Y|%BK6=Yc^dr1#K*c-1ZL9f$*VwIe{!ly%))gh0zAgb^$sa zuDk`t=o8Bv>8F7+01cdxx)e+4fFPT}9}QX|=;Lw;`YmSYwM}QCBcm^gLuvxO(#KwY z17q#vW1ktk8{>+Z9=uz8jng)0L9IIDIm*#VRje`jZJ&DTl-~UPFY@doEGt7k}IV;2{qCD64$en(kb|M-!4to@y?C~@NSxa zcy6@)5a!=C!29=Y`x<}S^5Pd$N>gj7ZH!D{?%|w}_&h%+Bng7ke=wDSr7K%K^U;`e zVkTC;R>l=u?aW!4?{ZWmf3;S7Dq6`+ElW!ScRUj3!z-U-&fq&xmnHZP!n@vAqA~P! zpIp+P#ssnV*PyY=bIt*$)yZWe!mwm__||fp`TE2~+ArgV6As(`Z>S_ip(fVhR&!|JGL!(lJe zVz-uxQJ3%9dHKtiY2Up1Dsy1oY}P`&>E4Cd^W7^5-|o}*eBbW#%`4O$BHr#@2%Ow2 zt?U0Bbha5dNmq`e?KIBc&(^+<4Zy%kf4BC{x7RXpue|uBm)LCZmObDt&#G&2-ZMx@ zt4$8Wz5xrf98@?7Tkd*~C}vOOmUS3VYVR{pMOG z{aCF|NP02%&042sq##`xfdyBB9SQ96IIc2*EZ&4(v0};e%?~M` zS1s<PG@%EzsU^FjIrjiy*X#j0eF8_JgrP zIu(I(Z3IDJq+61TBTdA*g0nd#gR{lq?b!76==AiEUf+*I4*jp^8XXhF%e#Q@E7e~CF1s+Q zwl*RZLLESo#3snGqO5Q^3g;q_qp*h)t|Z)S7F1bQX9}uHqOdBrWH{sEfG$fglCbFT zILJ#n)~@?G6UZ!@qGy0+G9Q>HaHPaC_ft!*3T%0A2$S-JV&Siih@#>9y;#%^zG+fMvTUM$W> z{g&WEQad;W=V{;P9`qjjX=OMF6+UE3 z_P(gF@NJF0k?Dt)r$-eQjT%){I7$hb-qJFC?Djcxwy&i-#r5^X&kUF&X>XbZzK{H0 zd>|;X5~OA>auK5$U~nJ+8NDADce7bHCPGEfu*}fe$__2&X>N z`QWL~e0=bSKX?CFD)RhVum9_Ddi`II3)KHZpKAqDk#~V!|Ns6@um6947pVV-K9~1O zMarvs{aT}5zt$M2|6I@25UI#ALQ;X>C-k?yPte;~gI+t^Gt8OS)MmW51$61;RGAop z#6{-dD;*>Sv^X<-n+axdIuV>qh9y8nMp~*Y6_wX~q5b;6e(jAv*OYg%>)8lai;!$r z-E#(}B7o?xm6F|zp~ zs#${+Wv6MXJ=*U zdg1`zE#PBphcXrWd0LnSZbT|zkVT;d91RBI=~g>w8Hm4w#n1}lEJyq*S{U8+Tf4fh z8Py!Ud3@8x=qtt$UmP=|xp@ZO?8AT0vmNO>+fi=j40tTDVn5G)P4z-jdztt!WTzhK zZI%uiV(b`8_tB8|CgO8_%H1yhwSPH60?NIpJ`KshAvD&G2@Mvp{}zz^JEX5nVId_{ zkg5hDOHmNTu?xC z>eFUfLwnbR5Z}6_eiFb($YS8-UBroE?C`rl2j&}J8`TQY8T>L~OnBcjPqJgK2gI81*l$I%DM|4$JcGeZUs%YMqW$4d~w z!RHLl;1~&(IKB?d8d#`e+AvDo78(pQRU~!5avi73My7cKC910Sz7m)^;+395=~v5* z?VE;?8wZ{0&S`Y?=E^O>w?yZf>yNXUt7L zWjMd>orCqa2mJOV+A;PantvaH_4fxpcRz{s_e=11)KC1~{GHYr{d-`YiL|Ja5zx-0 zb9eH6CYLQ$IW0AVj&_5OtzNqm$d)4O6SLN3= zCMP%6QM+0R)@j;r0SHpE|53=a>2U!C3pX<2__$&r|!3KQ6 zCiu?zd=1`zKyN?z=X(3WKM%C8p5wXY6xQ8L{rym+2ca{_(4m3%&+&YQe5-UoosIWD z109_yr=0V-8YbcN6>`dFXUHi~w-5Xb^aCpz=|E5v_EbiUE=)%-C(IhQg-*T&3cLx0 zY*aLzXcGYii`WY@@mr6@qdX+d+AGacTUnv@I(u3hiL#&-qfXE)`a<6=zm^UJ@6h`; zfJ}0Z=Y0O&&h@8%mnc#w2oU@KZ0!Fx1>gqHAf`$iLS0XbVgxr2C&U+BoWz&fT))$_ zy_$wi-4)saHtVIAs4l-Qz`(&d>yrA_r5xApVY5*ghixwE8>@T_y?`P65jGK%2*Q9V zR#S!^k%&N((iFRuEV7Pi%e=yrM~66ao$PL<{cBcWPl37!Fb3+(<9ss>2L=OpKcj50 z5FC}^8!SnR5Xs8OaqQ7VRI0VH8?ih3SbH-)EqLQM;K!@Mk7YAI<{nLuV_;4MMFt%Q z2_3+PMEn?bOi15?t#x01oQVc>L8cVk)AQznEq2c#@yh6p`mSZG+&FvKRm6{nZX&V5 z)E7T~d@G>|KT$EMbd0Iv2TVUxDpi1>InG?ikVmHgqd$@t2dAa8{BH)X6G%TJ4ixOr z{B_#70TirTj!pIouo&8piWB9B z=>kL0G?U-1h4P2C^}0SLv38Jtu1@~!IyLHtO^9G?kMWCvP@>RUlZnkfSGe13uFIYY zFN=yRwY~W6v}YqrqI1ihKdHQ~JrQS1wm$eKtB;S)KlC=IhX8ED3AUqQx>7+01{VA# zSR(arz>=YLJZg<00~I}>At|GDFeJky-cb*P}Z8 zT6D-^CEgJ6TkmI(uHF?&PPgg2^>mxgTR~5&v<+u# zN3B+9=}KWTNxLS5$+g3l6u3LQ;)T}nH{P&wRO1tCS3fdiT+j74PMY{Udu7YSQQ5BS z(KENS4$rB`fj*AHw!E%(fwN&tQ5zi}4!+ii_NndF>D&ebyFeAp#_7Zbax}Xt$o>tI zx7w9`TU$$4+o&rKZ@ge=!k9}tns#rgT=&p*4dTU_GgKu!-IhN%rKxg9!q9>s)m|L! zjBFi_YQ0`xo1z+m@0d5e7^4T(7$F%U=I3y1v{G*)(fMH!umnSZb+v%OKdUL3iH-SW zMT`R<@MJpJ7gUKK*(-2tf%_KB&1NDaBPlm6P;O4OPsML*P-wEwIH@~@q zG{tpxO@0U8@pww=nla%Cp(7{PZJpY*WN6ZY3oYT6<{wpOk4~9zVO8qdF=26`qgv_* zdZ^r+RF5OvBwNu)*9FP-L9Egm9y+p2dpvnGZm0CS?e+HddLHsV=!G7OGGV|y7xI&M z>|5U)mq6>?UYB}3RyV7VEsW(@8wPa1@7Bj}}$2gfnOOWpu7;&(ph{Y!@K zX$tm18MSGcNstAxix@!TmYGf(pC~@QMnB?gSN!8@Kj^ETZ&NSHs5b!4;s=w2zY^5H z*jGOq(Ccp`reZev(d%c%KG{Y%b7s1iT=CTw-U6X&Gr6zmSw28O`p|Rct z+66*A*2EFQg*4vd4+ldZofd2nl~6DeRbfklmV~@rk?i$L)HG`+{DP+DjP8$sF!`E8#bzH2j`hFwH zm7V2Ej>}E8XW0@fa&ugXae1kE6T%8Jn_MMT@kkt)mKPUamF297kIkp+gqniNrmVuS zNqH%G+7piKEJs3GdLq(A;E!u~Nk?s7@{shjAt`w|sq~qiUE5JI+?5KOivoG3wnmM@ z8rrJkiTt#O@IhS#s~kzZ5S&-yhTGCS%owRTu@)pjq7*kM?DsHWV#c{ksReG2F1u$( zXU*}s_*J70YG1*qPjNr)xk>xddlVu`un+;BHrTf=B9B4TpztsYYeYU?$l4-QW;`f0 zGyKwGVj?4B2FDDJv_@JJ{PCSLGhH^t1^Y}FUAgdUvC$vxs?NIT-P*mk7SW!NE-e-JDSzef220$I$7b=pF7EsI zZqd8$r=TnQRQV9^8oEyI`a}{pNwI!&P|P`myN!;T4pnQN6T(z z@3MEBwd@_u?0qf6_~u|IQG(>GOilK#=lHR~rYJg0C7UAV>m+cbn65qJOekVUOkesS z&yP}$Nans(OYnwZkDDVbBiei{gk02Mh)!WyQW0cS1a+*!NvFJp%nz|yBilq_K5!!m ztuSllMPNC0mJ%~-BY~0}A;)fY#5x^8#^Iylx9rR0%nMw{Rmyo0gs$@PQZOaDKuBrr zl?40L30Za7l88S|QyuZmG0FMqB_%Dz#rA^am@9@%$!pqJK6>!r(d8SO@}~?*7;P^Y zQ}1kOaEu;XU>_Zi@x~dtH%6(3PSRCdX5nkn*}(4L9V2b}5Mc-wa%+Y?2*xjlk~>ph zwS6jeL$m&QcJr{Z(bniu6*bM^4-@hnwSx)=4R_>C3R9}V7m6K@V(@}tl}`Lr4ny4p zleXR)1Kv4AsOD>BQ5-{55X3G3*B6BikfMdn6!C&7Gx4Et(x90s{+0APvJSyJ1<;ex#I zs(8rTg3U)O1`dUNouB_z>p=ukZG;odX6zO(ugkd1n+mcaf* z)u0~e^bd=NloHgF0$W8LU+d(<(CL&K25P#2 zKDSzhKLWm*+-CgF`KSm`0W#EtTn??hH;RYcgCSQrfu}R{us{5{`xNlG8uAM2r}oU> z!7q*Pz`^&9bsw8nE*?6)-?REELIu^SEyH+yq4(iT9qSK&F29GdYeW471IeDqRdSB! zYNL2aeOXfRem~jM(|WdNgd4R-JY*p+tMkQ;CEAZ0y$;|XuoJF=!o)*CJN3StDKCEa zbI3;y?NwnTr~U5>lMUmw@$~+m5Jar^TI3TZ5-c5Q0)^0~hGL3Nk^7G#m~dhk@7gs*ik(=$|BdieR0LUupPND|8s)ZxjL_ ztyg)Sx`*v|_L=IU{)*)q9D<&(o z->a=~7?t_jO4h>JKj~LK)ug=*8#E|;_Oj(_m-0BTD-CC`4Yi|?bPj3Pn_v%v*aSfv zAjc~*=@D=&%YrQ2d3C_yjCN$%2>-!w4Dz+f;IE}GoT&q_kTbR4gPPMHyUy!XGv(VM4@L?VwWV}~3n^p+ z$xK%RJFs!HEW)}Exe94ZKQm}lJ3xf+YZ8}OppInZVb0Z$X6+D2G$X^|wENVJJb)LS&l^^S7E=mY zhly;>!=$}9az*{9m34J1N7b(w**UqUX3CVBn#t^tkF?#LKGIfZ)vc(*|N51qMy;%` znKrGaYUA)Mo@v>P>)X4L;LD?POtnePPMy(R_}1I^;uI17s1hJn5jR zPAf9jzfWLK8QN!`Fkl>o>p%q(JnjY zt<-6k`l1vuQT@%o-aZpe?W;vRf8N9yl94~p+a!J~Y*Lb8BWCoE6bone&+qdi@y-<) zor4RU`e^ft-T5=ho7!jW6u)iCTRW*fw|vB55;+|d{?F;H8(9638zz{Qi3fAu^35lX1GGk#2g5BY5l%ua>W8P?@%%zNCw0T z{aX8xpF@mXp$#>CsYi#1Ww57>3(u|o{+j!aYvL{78N3|-#B$#IlswD(7d6}9e8zr1 zJ=Owt$5l??dU1YOQY32IXBjvbLeMs&Tn8 zs%BYT$EZ;qb<1jM7Z1Ons_KH_i^rEwpMhME<45Gpm|pH0KaB>Hc~*K(X_Xkuh;va` zOjN92s%H9d7*P~UwaVf-)t5DwO=vFmoGP7FJ#)p_hN0CYP%xnle_jz^0S%vnf@7lM z{Pp{XTMyG%%3fJ@VD{_-tCnosxTJkwSJ&O`SFB&QY$MH^S>p5TC1p4G!W^$AKCkly z@p<2x=2!P&nP2lEEP0p9;ujD;UooKX5`3^Fhxzl-6G!~|L^xIOwHbi_Eo(BqQL ze6-$fI31M zD;Kza!|~H8hO;LT_bZm-R9K_YI5k79E@KnxO+jq!Y1sH!XKQLgLfh(_4@+fJ zE+})R+ATSQi|05e6ww~9%lo0xuY942TVCKO86i$Y(deN$pi2kZ_4s~Y<-2eCv~EG= zo=G{i!5Kr!hZc0WAMSq#_<*l1^57D#VsNsFh%M0SQh#u4k27&ll_V}8lu}2Q2@@UF zj-h>EB~X2ftsheJP4?umo(nt5Qd^yti1EXk*Z2JoE&7q0lZ&Du_$D}Ci8(&8g+uN}dhxrk`i@u1;i|6w$ymcYz2%;ww?XKtTT6_|N zE>R**1qj@5XEm!Tc+^0I_|ew4x1ZLb(Dnfm(NhCBmT@@FyRKzzZLhbt6I9w=Y%X|3 zt~UYELmmaMz)tJT7$_cxX4N(VZvVLoc(_cdI1vsOYr2H3Ka&xXjQK;7`FxQ|q4Gx{ z5N!1A9>jU+O?~7*Dl~vXkejY5cpuAUU)Rgq-UaCtOU)FEK^V=nfXGU`Re60 zmh;-hpd;-^@Spvh{vM%vbEV(07m$lFRHz$&*ap3@4fW~UZ)w{E16cVQa$=5w1cLe( zmt3SkHI3h2Y+F~!<-hMm&%j@Xx?clQ>hdf8ysJsH)fw!xM?WJn?h(KN&r&+yo zIPPquBfxcqY!@o<83R$d8@v*8EB>(G&2n{;y987#~-FJ)fAgWU>9QY_`=Hp?~*Vn z5!oFu;_y1__n_~QsG<)=nzj)gW^HaXNpn)TRQOyxBEEyR3@^EXD+5OT(gY?J{z1=n z*U>_f$zOZL=i8=W!^_hW(@nI>{My28r&kJ9-ocg?l?bW&ncKI-95jvCqB<%8G zAMBpVTC_G9v~ALsB1AXLFo7Cw1nP?)Xa=5v7Z5VE$*qSDwQA8lt(+HJD!e2Pm7p}E zRSa)a>|2_&q#uTQ7%?NTvcX{#1uUqJykQ(%?OGbe{;c(|Rjn`mv-NAw3&80BaMAt< z8fH8st^yM=a22HX>c7;P-zI@H>FNZZ%LE~HrG0l0j!>i4NDjn|bX4%K>ETiGi1 z)IY`Quen@i)Q7VOI!R-~#lmqhL;Mt*0-r7)fazX9+@2M5h9udbT%YZ!LrN&*&A!VK(`7OO+0v5$)qb zTAfM2ka=`PJxKV~=9@Erim8lftrTO3$3~*~Nb)St?^bJnS}kWOv$S8a3j7}8@RfMc zjzH}gct9K)5q$-N=fL7aVDkVF111L3Gve_5ADP-zNP`bOK3H<4nBM!_)huCk=Nfj1 zTBUwzw?b3!Yr?y}rcee5#F?Td1L*NJh`e5Cum6327GhF`*m!&QNk`Ds7niaUQyc#E zmfEoC>ymkZsd5@$dP3Ia{*mVo`mS0wT4&Iy`=pxH)Y6mMuhEt$;Pgp)Qn?DOI8r!d zSchRWzzKwlaFdKFNhFl!VxKf`0<<7<2cn#hKRg5=r5gn;8@{U~Z8cK^C3REmMZPzD zHJ+nLr#unaC~8BAH1LDiy*O_HT@Y=6|L{-g;YptCNm6%+{s9J)qJ7MMboxsDlhd=O zN-Nc#7RsRKS#!q$ikAc0^iBXiU`bsoK|b zJ7b%p9Eg(@#^N6n)lpanH7(!!4&aUu{C*!87TrIGk37v+aXyWFYq~YYx56OSn9BwQ z8YnsjS{IKtI(C2MurBTN>^0h<;iW;lD(YB<_UA9jCfd`fU8a4E8e)VAgqz1=z>dkz zh|mHif~6LwaA<|{q6ol-NZ2r?j5OV!`B-cmk3Po{z)f=KFd$$+fcqG;5R z4trCrE2<<)J7RSAnxnjK z{hA{CaQ642>+;s_s%p)+IB&wuv2`|WVti?5N8`G-`1FlaH>O6Cj%AycFLnxLSj+ct zzG8d_zR_;j+1s^7r%S9#?lh9NBa|zC2)TS^mMmC{)!MvI9+B!ykS@oL1}076)!{x5q2vmO!td1 zdJUo?n=F@hOGs;O)`YH9YjU)JbFyNrlA%;q=o~sm52Fl3*kZXBlMbcz zK-an!*$70UU7TH>os^k`|7DZPE}E3&Ov3*hS5B&JNV(JyKW+51^3NTaHXCi>GMp)? zv55&oawBcloDBPrNE=WwXh=#%_TYrrgxL5&LmMDRxJA!Lo@GKe*n%NXIp8JXS*HD= zi6u_<>~PQufidWP#q(36T%-UF(OFN-kM|Os94-*(tND<|=?e`&)tOI#QT3@N7EjVEm}5MPuMG12<@ue3v%DX0yspJWc& z%1T?cuUW{kV|*;#uV``Fd%!Q*nH2M}bfR_IS1iKh#o{_0AHKbl9uydMz5a;{0}%;Q zpZ1>TpJMm~ZJQRymTMoq&3(JP5n>`6CLJVC2VQfsHH?y(sAtkh4%gIjUaQ+`mcvLF zq)RJ~iw#pU$5&6BbUf+n!sMFa#nIy5OXt_Nj=FK0a4EuJ`g66}y(M0bWNCy>qI zoH=bA=^~_lmwqZ4rTe81O}E+7>31l7DAVXKYGmFM_;VVfM7U6)=cGW=EnPi^v<6Vx zL~(N0^>v)~kQ*fw;e{0Bmk5)B<8n)N@jFe=Jiw@8_6+^?wMgT@yua{j%*}p>2$t9oM*$ zMg7xg-wXZDS>AV9a-dDJuxEmDz}SK^ag8hBIW`DS`t?6d0sInx2kdY_fJIJbx>bpY zD5U=DpY!nrm)b%7Q^(;6HjSNTL<1Nr(zOM~YS{N0E9=?DicJfQ7RHKAVQwjv_mz*8 z^}EK3&F?o@e7v7$txS{G1n^}=FV8w&Y*oJj<8>ToEvygD`_|EK%-9-Z&;(=9bT~|O z6Do7OSHM>cc!x`sGqW%})EZMjbQ2O|EescbfiJl9Ou-jC((!J_$GOK&JT~Cii4U^V zQZn$7Qaj{~Ic1(uXB{N*LH{6OjKqV?EwBEbV+5G`PADBW;(gFly!<`KNc@E{MhA^a zrH_$mUJZcX(vYaQxLlf7{8P@r7hLvg`lpUxGp!6zN53(WhYUDICVZTGjAWW@XU;Fg zw>T|d@eYaaN@w~qC$Zjo&h)`+H^H0E}T5-vy=${5|K)=2@FaU1KEwOR7 z9O65%wz!-i`a?f=7BC=6N(g9sR~gS)-4QTj(=JCR!Pwj(%gLtTM)ETAz7j z!p8t(lx9^UA_@o}{8O2aFSyhm)IasUoKV06`8xP7&m$$k?3nifR6A8T2)(^ee`Xf# z(#Z$SSV{RpI<>@ZFQMP5`bP<^Z4a=q!c_LvchzCG-|^SMZu3=wI>a7fs#JB}b%>vs z>X3dbOqH)X?>gjsQys9enF{=V2R;Vui*m712lh`sS;ZN0FT`mzYTy0C#{H?RMKrhju&;bS_>Y{W{zr`av&09*?O6MK^*`z>FCG+kVh!`%f6Q0E zc%RsVb;)=CsBynPetzmJ-w*y{XQ}^jyb0$;Xd~R@E_@OKT!W8eeRzyWGZI}<>N8IW5@!|kjMJmf8JQboFPa1 z+z;UAr+w}R;HS;xK=nV~=l%&pesG37(dYgzjCIc$@?@X;0sQ~0&;0;?e%|MP0RE@? z+z-_Mbf5dawbB{#jQKvU`H#+-p_vLSy zpvHdB`|z7gP)EP#edd2AMyucRKJzgX*z5b8l4%^^{2u)i2tH%}h_FilJ}&|PJ;3>| z{`^q?=Ya13=ZE_9NBchqdjX9dfDOUpT6`UIC1N4b;J6HMV&$ImT^DO_|d6s{6 zfbwQK_Vu?vJySmiD%XEroUL5{`E|B({pab~%JrG=&@481kE+X&+tmuMj7DK%?f4o! zGBJb{qD3a`!Efxxt4ZBb>3?;`Kr@DOY zZ?)g=->3cIw`(tdmnGkQ4@-XMvY8h=u&z`$? zdTz5Vc~1S#Nk9Jm72O}*HGWg8`0Cx-NAIm$_a5Hg&yr8Bz2cM3&s)h=(V`|BX-v3-;h8q0qwP>;oSnLSfL^M+;?6jg5hwz?hLE z%-K_u5@VnP4@DRnW{rTQfa1$$Y2EN6>2mf^K9sjd^`~Tc=+t)jGe4U%=qNolow#2ybJYsF@ znLBUSoam`BI~Oh58I!!|=-;lr_RFJPT}QvX_S(N4T{P_$rrdEmlW)02bKic4)_aS1 z8?S<@F@Cj`jhiP=-rN||wrLY=s0YB`2w`UJwC}*%qVK@lh~dL({v&w9vH5u->*wNq zo*DlVhO}t|;>Ax_!SE62kDp+ot6(8W3=aDLgRb;|Gdyi0odKL5s#ZqN*)?x&Pn-1z zkkvB=Rn6YKnQ|nI_I{@BR4x}1gaQFsQYm%<%qGFD=%k6PeT*Y0SbS!ur9=h`33`@P zM*Jw2^Qf3~%%%XJs$e+Rv#0B4e#E1sm-c{9EMIY4^X{pdp1()ah*vOi&$vt3(fJSk zanq(x9-2S@p-(n#`r||MmCIMYfO5PLW4zcN#*VL$JeBm=R2)MO_^%KS$~Vd{!b&2A z+be;Kz&rq1z&OXi2RH^lEo~wH7qicc3hn@_9=|gj@k?+AA0y?#eZ;GGf3a=b7rU?j z`_7$zzy9(QYu22&{IVCiyI(lC<8Rkr|F@kx>FxTJTF=ZGJyWOl%$V6T z?d;uP$7WKYnKO1zow^$U&kUr^9Hdblx{alQ5ua`D@ z{)nWlczrBn_-@#QtK{{H6Em_?J0lV+dKoN|`KZ{feX9NS-L=wj?NeqGWse(mcyeJ^ zz78>sRG)?V+3GPOx-v^9Ri>@G|Gw@;O4R-LGXa~oe)vW(A|(0ii*G%gyG4REj+Dtt zrLlzSxeQ8~dIvl3?T2cbHp?M`iJJRmtHtVvJbr#;k-;LFaBy>8CU(EO_FWbyANIIK znc*-I{qek@t@Exxe^U5)=m78gj1qVqE^S@+qVD^&7rbbgR2ine@EFjM6j)jhuN2y+ zM>aDBQI{Ax7oa2FHzdtG=%**8iTBWR5dA&~Pp9H!ud-~opd zTnjRmV8zgFI|hi}YC}oN(M=xql1|0}AU=0zNx$81P;>wiEqlHAu0h@M3jt_)t2v!- zVJVii(jm|}>{&2|hz(6vOosiVe*Q}MnPeM8lZT}#7Gy=UDQT9qG*cY@c_U^XY-i#& zCbmNf#N5|wk9%Xidod=ktONsusRryx8ew8Lj!Xm*vjrbKB_rwvRwxv&1Pe|}Y+7ul zEA}%@+_L4+q{a(ZjpgIxp2zEJg(<(iHubEv$buZG1$F!sRs5Kbui&{=UI~07dK{;b zbM@6n`slBt+=2eeE8XqvM!V-3vDU6FmG)_S@1FhbmTza@&EXUJ;1MGUJQ6QB6OTrY z1C~kQWgRYNS<8vTKN-X{O=~~6{$RVN>3Dn+c)S((an{-~FAO|lM05*P(+EXz;$+-n z+NSluKuX)9JvkOgce)9upz?zbgN=$L_G=lt;J zy+4!g*Gff|>ZkV1-vPVv9b;$MK!LL(E+;0aWaN;TPGbz;8sLXuCcf&6ivoN&TuQ8x z=|J4+&Dv7CSnGMl&Te!+vDQC;)TXCQy5EcV-UI_%quZ#kH9mjBANCyh_R$CYA!{?i zUvzs7&NMN$XN;vyXPrbPz6lqg6=Ed5iDxhvoof6uP^TW9erO)T|IufK5)8jA5K$1g zbt43=Gn>XL=_7tvpkOYhl@m2>-Li#UM)X8;5+sFJHwdpHejReM;N&v1HX{_hky&Fn zg*sNwthYY{K|HlgOX=ys?6A?7X5}xQR~Kr#7MfTIt1R_ol<~GcMO$vnsBq#am|;vt zXx7jWv^4+`z@jEx4v!^{LhW!OKt9Jzhjetgwrjaot^7r6#?nJ`@37={c`_=Q$|v16 zntn8=DY51o;Ef+cwzE>3QTnhU@}4zXA2>wb!_fALKQ-{c@xHG0Jmp~5WALtbc%BmL z9NG#D-U^3U=dti!^NHVQ2w6GrJz5ZkXd9)RdWCOfmYtZ zAeg!$tQky|F|(+CD_@8$nOM*>DHpMYR`ugX)pOa+)1AbDqmUz;!ov`4naujQ?It@ZTq@R zjc4lJCJ@}W`noQ45)ixJ?7iU4H{~02VEjlRh;n-f6Tk&)^Bz|ZC}H3gh!<(ZRkT_p z7nWD`z^vYbN80{AU!0SzwI|Jzr)llkV%z+`wH?vlorQPz_k9;Eow=CgQuojI+?~z# z^&Z4)N7@|pk~|Idv3>BFLA&O6(SPh4q)aOq0@ed{|KHn=@E57QIr2fz{yj$&` zE`ubHX@$w4dVn|KxqCk9kf+J0=SbV%=8J9UfjkZ4;DWAcg#0>aD4JGo#E=G!htf#+ zSflB@-|ixoYlBmX)glUM(4ryGda4i=bNy`TuM?h`9ihFgy*;=zLEd#ai(}Erlce4` zT4V?-W0NJZEk*marjbXMxy{Pjh%j0XeQh@Se7k{^4#2X6JR^B5J^FS5n>h1P9SUMq zk}Fz3cm+xfnf%Au?l=O2<*^(c4DGMlm-6eA+{zsGj}Yw@?LmTrm5>J(axZ(o zk`F6`DRbi^iDYwzNV1}Y97-_6LI@~VvS_P_j&>n)HrES z4C$t}3wO%T%a0*9rxkItGlh}D@Y?F+q=cw&3t~UBp%wWM860WxF+mL3tR;!`GMJF? z6zdn6%$j0Nq}k`ni~@i*xD~Oq%nDZ;qCQ(AM6e=sz$W=`sj6s!j?U;A6m}JQ$Mz2n zO-hA5V3E|}-X0$oEp^AMQMT~pGSjO<0@cN(K zY@IwMDtv#mRHHaDqmS4TuEkJT$v|a&34K*X-!b59OYi7-;_=+_(XgA<>;BO zG^s-@sb$uVU$~C2Q4fU1M~grYq&~t=rQGDlPi1X+06PJ^AXVbO1ux&f``^El=!YoS zL0b>NN6IQV5mz;BaJWRdnlUDE(Lg@5Tb2SjlMhsd02-sgQbWYK<*iL+Kl$CMHNp8c z!-~>&{rsh;#{4{P($r+z)hzUXepO$xv@&(U-`$_fqf_&vw`-sEzFb$G;a|eC$dD=g;7d1Loh4j`Rw=SVUsI46q!0)K)lENJ2KS6(t z&nD2Or2u@5hgcJMrz6DgKLnn{%2sF3!!s1eh^+Wk*IiwBZFF1w;5|$pKcq1wY3xG} zeE%^LUr4hq`o&#E&cTDjGS=O2L$wwgo*ZPqndPsJinpXS{PZ=9A@uO-6nJMP3hAWt zOG`~jPKu4OsKI>tZ~%AdguFNy`won|K@oj%N3+3% zJ!E+z++j(?a%h!){-pNDJJN>EDNfq*#Vt2~xHe>A#EqbH;ieqzoV%+pqg(e^qJx*oL2JC+`)Pv-wf`I^NaNX?%gZDF*KHYO_r+V(*XDMwp2Y zANrUO01`@FV2{TM4B!t(AcR?FwPEp<#HXKRLkP%{Apr8Gk2sKQ5cA%Y!)6z1jyIH$ z2eh|zDDP^x_w@Gb{`VdoO6IuNvq!s~pk%+J;S<&fPs#;yB`ow|(}zt9#53}#A;&zP z*2r)}CfyT6YSPci1=^|`wN*W=dkR}ESGqSzSGez%=Af=Fsa0C6mVz4?@j}m;BS^24 zNsF}yXK4@4VHwP!mU>dek3Gx9JsiGLEsQW?sZbg2b!r^EZS@;V zee*_5(5SN2=zU-;U52K04ePm);plPO6z#evF^-$0Iqthr(ma;u&ixGd2t;Xdn|4xr ze-4{6OKKJOUfO*oJbtnaC?F7~977tMRQ_1GurVd8=f1biCr00oud%Qcn z8<-_@MeifuUZd+=zP-lh7X;iTvoF1KP3=P!hZ@&p1f)Tn5fHq=Vk}OPj=FD}hXVi*er>>5x4cXH6X8qygz#nH7x?P#y^mEHI;twajw;@h z1AIQjpFi^LV+KAS8xWs*dkVFu<8!-q32^$rj?Bw0&)jh)K7H^pR!#^Nj{>1h?%hYl z$Hk+^`r#6%0FcQ*D!@pixZQJ!6r_Fdy!eMz;_sh#lS$JVxD2x>JU+hu4ql^I*?xFe zw~Bf0%k@v~gHz&xY2rPn+}FEz`!BuT(?4>5E&LN+u$iIVrwhM*4EwCsFYHD?l3+?o`1A4 z2g{g4y!!>;Xk`=U_N1_h{y{_R2dz%hq?1CJFlzi^?qdhm2T#?DunuzP=>-#jkA>t$ zLlH>u1ZuT@kKoxBL;3Zf5|Co;R@xt0;ASUjt#?fK92*ob#x9YzNlSXiUnGB;9KIPR zrXt(xeN^p&O|=S%Kj^WLAe=$Pl8g;JMwnr!{ftOqJXv@PPIvS z4|_9yA~{IGK_0Y_2%jrhRCF24`f5yEsGTfJ_;G;%2RTgAxi|-gQa3dv%b8M`T9`v# z-f_VQ>rc^zExxe2SSSi43N%c)pP2s;mCk_U;?$B_k6pX}v3vQmd#3b!C3Rf+{=r4T+QZAVf3omptR;BagYR9o>5o5|zu?d( zo+Um1*z==*>}tLK*_C^~>LD85@2$2(Dt8Nc@RqNxt;7@&6IBtZ=9|)()QDx)=O~Vl z$~g2_C_Ci$%)&uU> zX@3#FP~L!iox@HxhzqZDDjKUvmcwY0Y7sdgT=z;4LW#ZFLAo>tak7S;(9gc*p7xJZ~)J00N-B0<L9aOQXmH%ZRrd3spY%CedxP5S}Gqrd1BP zj(cu8U3&hGa#$!u;j+}I4IZgXe}bT!cDZDnd(6FX)<&W9S zAFpg&)6nOhb}zqo6&1%@<7&3j72C;XYWK2}bd6i`#aqYYs?j|Qud|tB)->XCPxscX z-Fr}RoA5HPU)qjIZWT&vi%haKxE9wd!I)N_iFhV75oECi!mU-dA}NVUknZ0kiB$cY z_3rqssbNn0&B82c75YYP7uDwbXC{;8-h$f%2P& zoQdbOexUS6`=XK!*oD!xbw~~!0&x|hGHr5f^-5>7XaJYN=fq4ep%{$mT0cZnP#NS}iBrjrj z!3H%-&@&(OJ4NZBQeWq0-LUbJX;&u#OE51)UVyr`qS zy<>FE@ZktQgSx#V*+`v9Cbf!sv32t0q*-b+|WP)Mp~*(Cox3Dh_R29wxJ;-QKXtj5=ylTWySs z-X_Flife$janvM0(2-?jIs_dG=cG`)NE&}?Q?c}KFG>NK-qo`XKRq|iudknfleV2* z^0YX0>C)+A${HFL)^I2y-Nwg@`dMCEg8q?38``9);S0xO;(CgVeg*4B*f({?g+rIkT$ z%WLRwFqkN1ZA}s=S4XyxFBA*q!cgQ09Vv_!#>0wcG6H4A??oz;-ma4k8*t=3FFdF^il`Dwn0{72`5T$#PY>JbfrD7ph{*tCk>mLrq+D z&5|4zJ3e=`6gMI^Zc*(p(KWnMEF0`9Taw3YP5BMd;CS2MMFlK=Lg5%GL5)vXRGN{V znmk0cD2ipe@7kiwDHXG zH#K3CT2fGuljC$EQMz#C;X@A|y#L;N?z-dlTYr4hjW=w&_L{3VuD^WEB^NDUwy0}i z=Yshi^V;UN&267EWA+T_a;Hq6GJV?A=E=>ITU(kYHBFj0p<#T(_{MQ#$BeEYRX4Jh z{3NTZhE}>NO3F*h%SsE13yO;hbMkZY^KzZp&g`sAM}|F}o6r~vaFcm$h#IWoR12F& zB`Rtn_P|ygnAl?_WNfhV_=prpx!mr6KF0hDvtwR{%F!?4I+iM?Uk9RwmzGEME5%Eq zCeo3Z+`*tG@uzv?Mmv<;jjNtvCleQ!N!nymTHDI>Pa_Xpsjh2|D(z%>CptS%Xm53% zP-|IU=Lye!C-BYmyQ)+FVW&=NBu(EXB?gwvH7H zCuL18oND7V8fP*51d|(CAc!keG75rs%Gk8SrBd#I@_W5-U zR-c{UFgDjYM(T1Gxr?P&<)RzzYuz+{WPN;IuB~cBV*d1sH47)*TfXA;OXqA@(`=u& z&V6CY@Daro)m7pf^n&ynx(Yp7zVikRJWM%nMazwTm+*KKF0 znQG^7cWQO)Zs-`Tq|b%^23A@@6yO^A43#u=h6W4u_R&4gCAcP;5`=Jfd}yyISikR= zynNW*+OsSTKJVt=5nA2<$KHFu$5CDT<8xVP{-CZEvRlxcSHWX-koWdsY_rQc#dNYdWfMTcx zl`wxrpf&*m$W@^`07kr8tX4~hV6j*+t;A}vQaz1Za8uru|6@m%$whfr`XA4~YX4=E zhP+5foub@}H&AaYIJ%}l?ZK-c9rsVhv)B`J9;ZmdMb-vAcs2ec!K${hrev_HHQFf2 zUW7#vmYPt}ETL7VZvdgFiB&DhP}HRAabpfeB@P&`;ohsg#|b>q|AQ~mmAGXzJ@o&s zu1WkC>YD##1@V7E%HuxQiL=FN^4qxe#te{=qzRR}>ZPh~x>n&j(XCGHkl*e&_6n{K zp3|WIM4W-?P=n8*1RO520O&o$VGl|E)LA0h^kH$vv5!00bNKL8^d#WY@IKSi-UrVa zCOOmI2U+q^z3A>xr^;_1+aWgd_i4l5j^SZM#L9Gl=*yXZ8wP}r5sA{7b z<8ts53@93riJ!t`WC1mVKEX5*0ksPBlcFp{@tH3}d^p3;O1B<;dcD{0WaFKF@A~!L zoDB8lj2y4}uhL)6DP%t9u`8Tjv5=kpLtB2n?T6|?_Gk9Bv+X%K_OsPFd+dRLeGht4 zFIB(#U}jEE=7Tu$UB{P-`%QM_?#xfw2oGk`06NWHv*^mo_KW+kdF7mzmUCXY<~w)Z z`JJyi_q}@bl1sjPWv{a{s6LMBlHaRG@`-h3;5ke6=Vb6&62kp7HUW<+MjHzxaVdYs z?YDo2o+s8`a`e@G&dz*v0rug43xl1`eXo20#*mK>h~JZsLHmxFr-aT$wHFjzmTU=B zm~_P^eovjmUQ~a@if%F8xctikyiXUNlb?Lfu)LwBy9b}6DQq!cZDV2em+ZT@ESGW%9*deZY=BX~I97qJfxc#oLV2#dER!V^5} zpX|+=HP0V0XO)%?w6qMAmS&kF{&};quBfVLy?t@%%B8vanR8shz><|EOKxwitXyLD z`AaXKF>YUZMVZ}RQBl5c-1I9-(MB6Deh~X9;`?b%G~f=>esswMB1s!?y=8$=^D?IQ zpiFxa{@5fmRe-7UgCVoUU*X4IXdBvQn)&{)&1hVtS)1Bk=s&bgg^*pe%Y(FosL54Y zvc9EdeMzay6r~;X(QX-b+rMa0KX%J#w^#7@zhXMQe}&y%M(;o4a(e$Im6ffxFDY5M zBoK7X$;{7Py0Ucf?X4Arw+D{-SaatX)*Lq0 zh(n($7;93$OWiNOYu4AGSK30X3E;i7wORCD0Ro;%sh=TUQa;n3D@3)m+Frzf z;4=1t&wN9q4+Hb7?OQP=-{nE!dTrlB-*<%PV*g`)?n9$K_o1QhE4;&8CwcD1$ahEqR+6t zA$00XrIWcHpFtfD&f!hdJoO8p-9`2d^^3GMgLnATk$8dbqdC{u6YYOwnPPg*SYs<3 z^*MRTHIt1sL+_}W*6??>jk-pet33z2$=5K2zEitz+Vh@}hsp`=C>IBx ztF1|Wt|p^~%N_H($>;u(?Hl>NS#C1kSKw>(d+|*=yOj6+X4<*vyoYz_k?&C=+Sw_Q zv^6&x-%Y%bz9$$-=qPMDQ{IP}R8>NaFeNdm8YTUTP{3rVs?;xuvIKBBO73MvS`QQw z!TkE5m>Cd%jFkxUNS(W_3+0JRBk)es}bWL2(^|G#_;bgFn?%%(z zvpRoPSaGyB&pboAN}ajUnju}FE;~o|TDPiKY+{dH&K_0l-X&`3Gfzv$x-X1Q3N4OB z9p(aeVR_=qZ_Tv$%&bm*)8@C#oavDZzx=}GkTTf)>=|`7$$ru?@;OO5a2=!RJgy$N zemM3_+L}(#DnM&Qu77yH&Mb9b+M26P@1V0G*PRw3Ie(5iBW=xf>>2GjnjQn`Bq8hg z1z7X1_)`$5>n(|#LKL(6Ve4gOpl=Xrs)2jO>_nO)PIi$W-jOJoU=T@WbSRc^pLNTq zkYafVN*aF+mq}~&rme%MmrPkM$d`k1Fbso8Q3AX;o~2nxDdz$U>5pB~87sbg>_m3b zzFJef_RA%@xnvW`T*}a_C8usI5uz3WFxY$LoUhJIcx$Rei%{w_NoJ`aFVg`~omm2s ziZ+;MWvHu=%oebi*`@$xY!>?%(>IC2Fi`p8)HR}L?h;Tj-mNdjMp2zz9j~mw_mzjq zy3Kam!Hk+Mm|_m{%_{hN%{olqI}1tf&Ft}m7}K)g0DH{j0K?7Nq-2N}sHd&UGdX+dL9?gH7v)sq)J$iR>>g8dkNi?Os9>y62>IxbNqef4$vnLl^k*5 zjyaE4fe@JmXg!4aLn6xnH%N2y(tZPQ``j*VVF{v)Of+NQIynU$Hbq41O<1V?h&OV_ zKrXSsaxP6SLy!^+2W*fkhybLo>q_Jo6{1sHq%d9-&kvGA45u8hf2w)4D^xEGCxVP< zXLbxoESrHJeW;EYpwH(SCRiMe|TD zV^+EaMJY0aN-;z~bb=Rxj0~m& z!u^V(Fo2v&jE)D*%=fp2iu8MBEbx*dE``#cOv|d)*t^HJjY-_m3)|RAAnr zzB2}?bM)N$o~2iJiPNpw%-n(chivPyLqC3z2nDGCfbH@3%@^r@*0w}ep3j{LAgj}+ zNI>G!3`V+eMFL7DFw1){O^3V)@wV({GJ>#b(;<4Q!5-ZSQB>HmUQEHQ^Ek*ps@y#U;yh zsa(lqN=Jw5%RlkmNy)uwa&vDYt1y5rD}pK8X@}5Y#_qzX8!$th?6u;UFT$WrRHIR& zLFg}>J@B+eCTuWFnA(d+UCLK=B?9oa1xhSLVfpMCPX8J1yl^=f=m!fS*Zo`0QFQ9T<{~zz_)LRg_IL_ia2xeuM7H{ zLa*meFZ@>e*L9d|suo;Ou)W$*03?QM04td~zp@_{lnAR2cxO^4` zfglFyrr%D#JLGPPg!xYle-@k7#U{IdF?(hsA^{h$M<`gZh3(v|Eam4HhFOeii2#PVW3Z>AH?kaJls9l&5SWS<{#NotLNSO;%= z+U(+uj(HAJbga1NoaxiAc;m`>mySPoVqxYA_4x(g@h!TyA%4+>vmRdFeM6w@j@QoE z`R-TyTO(_Z2D06vwg0-Q8?KvMB|aLNa`C?%-2K8f!*_=LWTu#b@$*<>2xIcM2YvF< zedjUw!O9qXT4lI@clAmA$l;$kxlcF-|4unYUrn);u-w!iO~%D?K}pAn{{%( zUH5~8WGv|9KKMTqH%W%6Jf@({zbqOIEf^t% z+IbY)@20dsT!fjHKe%LQ^r2Rm-<8O!s;a84s@4J!WmZc{00Mw01Av2M=}F=b{+vrs zG5p{u9835?iwXR9ia&_LWY{F%(`^z&d`vL@Bc-U0l%WeL;6t_uy1B_>KLr}dVy5Jo zW{?O%I79{s44T|n7zR{sh2L;*XkLXj9vbymKHsZ_S<6}bf|t%9qh#{M zuU$F!*4Fdb`Dfj;sQdbbiw`$c-w=>~m+fINL8fg;u}gMd{Hp`IpWikoer#!Eiss{> zApBC;eholueVOny!~2CD&%|n`SnA>QDP#vzMM^7NKtxP-VBD>uSZL(DIYR(wiIe%(?2VA7;44tmVtIn9u$pyWin0SbucO3Qtg7_PW_>-FQ9Q){!Ay zVsSFdGWE+V1M1E!2 z0vCYSrJ5C;qcKuItKe6P&0$xaPcnX{A13#Dva>yPHSABn{LbXQ`Z*UbS$_4svlhK{ z+l|*1E(|!Fl4Y^~=|fi?%AV@B#6d5YRoV&%-o!et**m+pEz3M^d)L5)vw~~3Z{OUH zOsBd^7BjXDtn4?b+W|=dE%}})KJCDtu*qMr7dh=x^J0_LkBBbD8e5+53m$vKnq-`F zGHi>~H48yUigTu}Db?tUbDNHXM7uqm-i5$OCWO-xOEQ>E@-eejoSVVWh!o)$jC;Xw z9JtzSzyZqJIpqN+(dcfQ-`Ef;5`+ok8WN2YYpUZ_vC^XYP%WaL1*PC;XOi}f0KK{z z$Q>I(eP0UoIc5$gz<-D1Q0N5Kc-;5h7hQuB_}-Mh2K@vUzjV8H0-31PR6n*X^zu_^ zF49lo(LQl?j-(7d1$Mf23>Wmhh;v{&)~)JCG=v`o=yCi*(Bp24n%v=RE8vSyu8wr7oAa6?OagUS`?dj`!($DUwm)!+WOg?GiNS(>GqqiE1aL_ zvsf4TpE|U6f5mL4!I<0fva)s;9{&LJm<)a56H0{Di7W&U z(G@ro+P(mI5hEby10$hBo{W(maKsskYY}cD4PU55n$rHZ3yX}V_FW0U@)=PQ+{k@VyK{BFKkv&OykT7M%2| z#PW9BSvNDAD#}IZ$hi(^O2I4(2sN1yDa4?t0rW`)`y&|BXTu&uAZkS}cM9?aValY| zagFuymg=Thq%@pgRZxXz=Y{;Z035f#g%2e8Q0~Wg6P&+M?+=DBX-ChG9Xr+q(<`_G z(VALZftL~xLKiJre(3P5W8Z3@nmD-WYu}&Md$oBU``$~p-gMnptc#YnKXoL?>PbNqe>XbC}q2~cPZ1$8SZeWp~goR^4d3(b2^va&0YQxX<1#3o!_QiX30 zmDT|z!8)1{2d@yT)Sa#gZK-W-uAHa7>TxS8O9r05p@Z1R z`kk+K-rC>VY8sCiD;g|sw{UT-esWum+oJAcmb@vumOpv@%I&*Wbp@^JGGKuXnn)jm zCZYdYnuLo%lUQ+exC-A~6e$KxNTu-%$fz{ov&D!e1_KdMaBJM2nmTDhe-4Q9irZsZ zQF7^f7l0(T3$Gr3$4%~8dFFAX@`uvo4W=x<_pB%HUD>*Q{uTRik7R=;4G@}t0!{J} zub+{aYUX#yVu~TJg8&$ye}c)f0XYmPY}}?kv=tc?l%%esB?T3QWu&M}L$ohD1VOs0 z)*|a942JKi&eyP=4!Wj}y?-ukCw$qv7kr5A?7yV$${Rhi0;$`PMIKYPsK#!($8Xhk zqewVk2{MlM@+!Jf7eU&cm#|ro$;xMfiK5R`$5tXB>ddy|dLS)l+CXChEV?*AO8_+C zGZ!g5{ET}+c+(TC>PiGdArCTXXf}c3lwhvA$WwrCrqHE*P-19OaHNe>gqk-0>`0qN zOSmvUM|ttFPxQ0Xb}P%OB1w1k_Tu#rO&E-8aol zKYG|;jYzlthOh??pze@!pKO|}*9|DtkWTNa8GOo_w~^{J6b09lcWhw5%_ zDu4`{)YjDAJh`B;ps~J&z?RWSQQ@FO%07Wa(l}E$TDWNo3Ly@-U`N3Vs)fmBgPI$sa-r{*MPHlrE)DLGUyYX0*aDsIk?(V>c!`whEwYz`GS z<8haD+XX%jHT4yRsp<`KA^4?Mm`EOunOTHNhPgNli8YMwu$U~O2}V6s6((A4fN=;c z28w>nFlNk3ACl^5v4|K60IT(Q^!VnwiS=!GSgo&QQPdZTkt%j4y~{!OhU+V|#Uf}f z*{-@Jr=@g~E+(v%x~VYk+#d38zB_+M^OEaVl85=e+&0zq(rq_gSG6(I=kA;J^r0&c z<+r&qOVk@qQ$(Lb{(&P0Y2vHTTe5y!M%@{`>-NnIu9L(a+qSN;h{rM=K2bSMdS}bP ziUFbvgIPWwv7vUzuMz=~AqT)FAqF5dC?LTtH9>Tf3^`vgl2w53@X(Mnp9w{UfgHCB z^D!(!2`f>Oc`EAW?elARc_6DXt&`#u9AEDg#W!#J-?x2BF!!Rwd9UpH z{Jkl&fYz$qAaDFiV9ot+o`GJh*B2M6f4(hHv*pd(exn}unX(+;RNohI3IF@Nb9>ea z-`S&NW@zsW{^NA!v5;k?OMt`uH?B)q3eHtq1MHncTl2se)_gkZ8j2n8=K!~s82(*q zE1kQs#&(tb8pk%_eUjLw`LxD(XCbPsNxpj)n#1z72L6`ctNH57oS!^ssB?G18G@74 z1pgTR9dJ%_9xsDlLlY{1UQ8Cm$P}|hLHle)QeaLfHn5)p$UDRJV#th|&GH&Vhf4^w zQ9t5`q`fEl6o4*1q_i6ws;gq~0SfcI<9y>XomNHgFb^R(7=SLd%{BT(jH0a{tO_^p zBzj^d-V<2RR*%6F#8&o7BlGk^sldc}T75Y;XKr3eX-!8UcjK%?f)$O(j2)9!<;bIEdJd8@kycgIj2rep4#4G?Tpro?21`#mi zl(9}I2^R$Y-fWlCA;(NHXaEh+;D+)Z!*0#u_Ejg@pqSg+d+fE7Z%V{HxIkXOv2Zbp z=miE26LwB2c}#Npen|4r3>bFugL|hFXgBqulk7$Vyp22gcEF_zkH2ra z%5)y|3+6}zAjL$)!dNvN1X3eZ(k02z95^Ek`7&dwG{%y_4k28UCP`&TQpCq1p~AdC zCdtw|RyUaQCpHA`LhG5QL{Yv0xr)k;;d1q@+djE(*Swo`N{Tdb>#{kW1}6@Yw*dyt zfWRMdA;60=+l!Dlz&NN1DIpaciw;c)V#?_6t-miaQOE|Y!6xAP41aYC z$Ho0~uMWCzs{Z`O3{a28SQ64V zCqI)9;)z7Oa?+&JhM!IP2HEQqDy!PsswyX_IYW+VguQC~H32qAOjwXWh)rS*fXtAbQpj43ek@mZ526oc%Pb1jy2# z-u21h#dopd?=F3C>p-3<;On1%_V>@VitO0&%;T@e7Dchcz9laN`W|{?+gF)#LolfR z^;)mfp1<4zS% zwVeC%*>eZ{0aM<(tq&~wE(_f))^B^`p}rCyb3dJx!8{+lv-0ZaSLNF?J=d~~VDJX@ zb8JDkS0-@Zw;x-A?<+6UY>1@qJ9usrg>TP!JOs_2-NG!kDrh7(1Osha2BbCRm$0Sz zFM(>M^VGd>9O6mZA12Y|$Q z07y%w#7Lh<61i=Bb5lu4adt^5sBE{O|1)nPoAYl+#8a4yU`=G9!fuGfN~is4 zvtIu)o7K-Li6!#$+A2e>_JORkI>${9_-bP%3xnB5pFZJ%y#ERd2J^CAfmmqC&Y;Wb zDfIl{^&#hjgI%azD9^y1aUv%H+n@+`7IY~%jqEJLRsx0^_+k>xnAitH%b*H*FT%PM z`zkXtvnjKwG~&iM8(+u6g5?{43kH<%vE^#>Qz7=3l%wc@Yewif3P%QBfIHc2+jB=o{*%_Co)f z``_B}D6^2||HV!A9N7}y{_Z^=s-Fd!HRl=Zc?x1J--XZS5kf*NQ3`+`V4s52N67;t z@Q&bFVXhZT+#v#~YGHN&a(So%v(I_2c!Z63^(Rg;Jf$}lg9HAHmstH1Gj9Idk>?6K z3NQKDo-==b?G+tnDOcK>i*^qkyZ`axmLJ}^sv8N8?pF>!^4bczvY0?1Tkt*wh;zb= zw&N|}9Zk@bSMF`IF-NBK*{Lk-+|6i>BT(3Ks&1=;!=99CEfcHlIHeq3>h zI^D_~e_T0qCCF!HU9{fLrG4u1t^}e_OlTXA^kM=Heage3AT_afJ82+BJ*&^)vOf|< zh)_?Z7__8hixx%H8s#*CGxR8y*PDgt%ph%L$Ks3X#+^0m^*<7Qrxv;0g_8={@8FjY zESudO4KxPWn?13k*M(2ZRhH)6bmh5j%%u-Ml0t1%uJRD$u_?PdI(9Fh@*39FrAYox z^+y{$8SJ-2``wq2yyd)2_$LQSemVZ>K`w7o>om1BGoU+E2$4b&`R_J|@H%?u%P=0Xunqz-YTeD=OHFRfBvacfTUL!WM z6B1q$^q&PkswH9qdcaUD4|UNd8^U-cF`qSmVJzIkh=ZZQDFa6osyWhU{Dm`>KnC=}e z)PP^cSy1jK47>B;L?%!ck{enCekUbwG?-0P09;5i@4%JCq+81d1(o*_4_;T!OGd`x z7?i1@zGB*P7(6FipFjZ%J$Y~`f1JSjQoe#ZJS3boRXpkCjfA{5(%)P*0u?UfYk%;N@=5Hw#+xpM2n{7ktgq zhg?)bZZ;k~ck|`#2oYYO$x)sfJi3iNBAYhsIeUc&*2TR^=VjROpl1{4S%To;!9*t2 z`IO~DDTg7{jHGEP25o?$Fxo~yF%VCU!HP% zbz`tG!qe))f`qb|=%tYe^FuDMxN~k}S#(`2oPOAL$4ZGkg57~un!K}$pbM8}g5um` zLUEh%d-FW~_fhTlBoD3Uaaotdzof_s>h2v8#=)M13xxJiCbT)xlW=H$k~9r`$mK;! z3Ie`dA83M};2vb+ZG(BWbQ@tL=@6@~JE?5Q0i1CZ$&lRN|D$~~VKRi|aXcd?yrOk$ zsfV=-Q)q`^Qv|`5Z1w^UQfz2V1zvJcl4&i{97$^6a#F&+A?2fr4Iw4HGs}rTn`qQN zX5LF*Ip)vd;^GYO-bkkuhcslHD}qt8T5gG(X{4aFRK5Xd)7r4mPt>Aia!)LlQ2a@G<@Pz$Kd`qP9v1G^%r+ltB0D&KZC57Y*Y+q`I(`jrFi6Ab|4{u?Q1JKcXDJlS#gLMbDTsT@d6373&*%2a)$(G^ zKTL^d4_$NT3D;b}pEJ@L$W-Zg%6A<$o@4l;$@d}qNqe6Mj5R~whx$Fx8i8sD@XkD* zy_naD>#=yMhfnsAz#e3@*H1q8LCg%GIsi8GyGM8~_Hv@4`_QP*1@=DueT8>~ zmrn9r)X(U2cvyI8UK@<_nVNR-An9k(Z^|m~ZGZuG$=(JUKMwN9r=vaxy$!T| z60Nm;4&@_`S>?Qj%+)Boue_r@2eja8G+tvzQRAP&YwW0Qm-F`<;WOZ)6Mu%sX|#8l zY`n`*{^V;oZ4k@R-*?y=@-6stfH!?=Xde_IJ&gQ<1O>o>x2wegX1@Ke? z-@u84`M!~oWE6nlweyr;)Jo^E%pyvJwHbX~S0WeLn#!^mrmi5SPMHm$L8B~6VH-YX zJFG!>5pF;uv3mHH^+l9aR6;!5fG%#R2G^4QX6eGa)X(o&d-j{VTo!0tsJ-K)P z`AQ~yE{aFL^4WLS9ofC+OEq7-^xm8M4lM~>$TEI>(aJdk-oPul`G-4C{bj5o|qDZ|U5?~h?O-4g&fZ<2}@N}SGjD0JJ zxS;#-Mg(591gwoGVDTv%2$b?M%YhSr3UQiPj0xjg8XIEuvHEz`P_k)p8cZiYIoTxm z=+S71I+&dr1;3ruIyuOG>zgq#L!CVqVyfRg^VZX{qS36=Zaq^vt`QdYf|iq^TKj-d zDxd}#yar$w#rfdzm=KjPKoW@Z@#L#GKZphbp`yIfU@5T*Vp9PY;K{urWTl&G^)P-T zMVG1UKr~0|&@vbk1rgZWr2b9)vq@UPHXRul_|3+yx6j}BvnMuP`jpf@EY~wQK|5}V8tcj~cGOGzOJQzbeV2&YEOKt{owIY?(AQvUU%`lrS-MCei z^9TxU?}5`{kJ~8>K%Pc(%BNACPoBnDKkmoJF>ai`T;5CHeoCK*vvQJ;;&t(^%)WMT z3ufL&lDw5Np)6WjRN%_y4oldTfpCM-HPXeqL!v7;?;44pQvGO-_nO=rX6_93ruqlF zj|3f}eqMDOTC{AF_V>8G4i5pM(x=ftgHVnK9TSs+iG(=aQ8RmY&)56w$7L2| zOb$)0zxnX;iznyLVg;W42QTfLzQo0v0)>lSd->5{_Z(P!+0st}lY8Rfgk-rRE9csQ zMIFVa)7i$}fz2`7ow+sR*PnOGWlIiLSlf3jy88vNntslYV$Y32M`EdqvS2vpVy0k> zG71ky%ZZMB-gUKA6_Ju6QdHTQ8A2mQ7;ABHDlX8@n+`G+szv8ppLW{Z=|+Zh;<3Li zyK?d6XK(w)rmp9AZ_STfP&eVs1rMBU3s`M+k(L8%p18DZ(_$9fy8SDGr|w&@bA~N` z z7xtB1`sA7eEu{^1Yo6`w2Ns+;p?bv>TfvqypY7WG^=*4DTXMy+zXt+@MX<#}*KA(W zRcsPZXPei}SXmjZvfj>YjRn&$IBoV#aodcY3+{W0%V4_8k8&A|Zd_aj+ci7du*bo~ z^VF9m;)5nks5?Eeb&x7Hu?;m?;Dn*sWSa=RzC%wcgFsN?RuQlOD-^udlA2y_LAU$q zn1b6pr)AD?ni;v1AvCK^-VCDPNHmj1LQ^q~kly(#G(y(bhB7oVc1+GpC*xo9<^xpa zuN$`{x+%J^@zL$^s5%;CT0a;+eCPF4pB0~ ztpHho7_(r;2yKPJ(#$L}euoI$nF3$|dsYk8`C(7#l#*=T?F`XUL`*>frO{V+LLLIz z*CjhIyawFU7`Nxx&q`sQ=Q<7M@h|F7k34>uO#5aEEb$#+XNH-K~^?pCd@XGP4?(ex+yOEGX zXOTF!2e+LCasG{MdoEpa#nPU{><*7-1ADW(EO7fH%X-U}PWlb8 zq=V^kB7q&BuO5`1293&ugI_~cCHyFjJYW}55w!wBi`hB=%nHIa82^C^8i+rj1_m({ zdBlsf8ZsR9h9_Ij+qp$bsGA$W0zMC+E}T}2P{zuLtK1q=Km)v-h*D;8&5~%Pa;S5B zwueX+MBUsjmJd?7R;+Llseb!&_2ANY6sG;vn-|JckS-odtz*AE{5+UNOwu6_l(;gD z60|o9Q34h2RvLU_!8CiQ@=-`pr7Z`ldMxX^5{1Da6M|(yl(vM6Lxm~y$YXhq6Vd}V zFlQQ+t#a#dBzpW@j1Vb~9Td;zgc+G2U%r3%dGU|no;93}G~O{6gD*Jm2nrfxe#vbn z@YHC(SBQ^{^@|7BkM?_8esX=(@i?ug?@|6?e9y`L-u~DKpI?0kF9p98JKhDAUA7zV_ET+FSzLB?SE0t4jm3FNTv7lYu7tN{5(Xj*;^snqtU;Kf^ar7K7CEPd- z`8)G`4V)St^DZ8G|M|Wan%eLZUDy{YY4m;NA;&FB+eg_}0Dmi8JmOy__k>cbk@lpq zDg>RjCsbIuJzK;x>_Jx4tv18}`$*lB!}0_5McNZJpC`MkNj<{Pp+MO9HJdK4=p2f1 z?Et$)Oa!QuEK!Ov(Ylh(tt6=c!_0H}(Q#m`?i$KH;hH8_Jf|D%F2PKr{6+FcFB(vXy| zr}o(xn525xXEjC`^6i6P$rT%fKh?+>Qr}=5f*7VSSf$7#RXK&gPZSmx7Ke&bn8-o> zk>IkNh>3uQ8_GyTZX&hn7SA~$GfA6zdqFXHP$Bgaem{y)3GtPLjo7M;Ic#M0&^2k& zS&PbEg^Y9#b26JluwfoeCbK*2{pqaMY_QrXeiGHJ)Hk4}y0Sc6S5haWGhu2q>j{_; z9xXy8%`{=75mWm-Bnd}mO5k3ju^a1}R`Yu|DtD@v%#K!7xOR!L+Asd{1ClG$FY*`g z=VHRF#B}tLU}2#9qo|0HDAofs6d)D=y`gjjaaM>Jc?j+u=UtfVce^qi!1n<4u9c1; zqCN=JF(uoWoH!l}w8v|6T%4TlJ67C<=> zen#h36mQu_D&8tX?~sz>{5-!GAYr>4H^uQ_33Ak5QA&5Wcua>~gAm?5;;dsIC6D~U zwId$+_9s3@gWAuHa}W84oO=K*pD->Zpq>{B!G8T6@^4f4UaKfdn+XmkH#-V0M$Z75 zM=W(NrbwolYH1OrO=n}1JYuw~Fi;V~@z8{FvJ}`klqk;2X$3_%iY=(4Y)mV^FITW{MoIa!Y2#Zi6j@ zBS+d1QVmUG%&=d~C`&K`)rumo!B`);M44Yvj$vmtJ@@&eEAQRVFg3L)+n3+m^PSV#Us4;z zV(ZB}O+CJ}JAL)2_iXUK{*So#B$v%MC->zXWnamAKTHm2wSjk`Hn34xmROup8;HT6 zlrtnmFs~0pGi!LTaND%`mbf0n3j`}GO2ZThK@vk5jN}hq4I?F^0P{{Yf_nCb3;kxT z!=sEA{}bLz%72@4 z&)s^hVE@_bXSYU5HBI1Psj7f33~K8U-`**&fzE9dDiX0E3L=pvm1%wxH9Lcu)eYw1 zsOQRZ(#+K^7fo`adeksYsG`|hyih|&^J=+@X35L{lJpqGf3U}G;x{6lH!R;ced0Nb z=A3`qf}M}7?mV=pIJUcQVcD8#SM6*#W6}7|s@!qXdj}3=Pnus=Ju4DgxP3|cdA;N2 zUAVC8?79A;#fcgFB5EbeY+Ny8!Oqgz%@xzf=QxY_K42&B#Jw&?KR&(>Y-BENfE0Hh zgqyeqbzLPgFurN~C`EH-4ONa(P5p)v2!LjbQ5wnJ$RXzxMyI{{d{Wb~$t7KP_N|}R zcGkjKdv02=^YOk_*Dluf(Lej9UG>}M?wk>5JaFKEoA07Dv|z{Lsps~No4bEe*V*&@ zMT-(M_eI!Sd>0FMMrJin?}}m*ntkyZ_7D@MCMJ`8K|plFt{7b+Gl2|?0;J8d(F9#H zEwp6HAj=JM8J9^EX2Stu${ecvfB2mG>wCCmL29O;Z|(Pw?tbS8k3nIn|) zXB-C~SAqpRCWeq$kH-`1VQ+1Wian8y6czM(y49+Ukz;pz+#6x;QE$H-z2}B{1KVW$ z8T~iNJ0>(LJFKqeKWH+O?}_8&e#LF22jq{2*&RdId^+kH>QBR;16-H>93zhenWR03 zY#hD@HqPwf-=%IP8XIe@Kg4r}+Bm2ONnHax)<|nw)7D%iEltNwK{gDYXPwZQSZ184 z)5S7nXC`3@29FjqC`UNj>S`oq>Z_Pu>tVc{x?-L)hS}DJOibpMF_x6H+ zbHTq^;NLoQOtZO}&0NI1GmU*+hDh%?G)-ofMP=kVX!afM`iI;6 z>XvY#YRek+{v6ahf&XPcYn?N`Dpx&#t3mOH;^QC7e*sNugzwIIoKgx-405zlH-AKS z_!yLwUmzku)nr62^sF_TY$g<*U@|63?KXfWlqMGX0sA*;13u71UwdXmUu*+J{Y0x1 z6@v?5SRY7PjUO>XCH2_=uxG_9V-XOv7)?Z@7&=3>qp-!2BAuv)pqWO;bV_qm%_sk5 z*YjI?dyDV6E`DZxq$;kqt%*JnGzHd5tNL?I!KdQex0J2^mpyFDO%F`zv@F{{YtyAw zIqFKEiDZeGn?mOs{PDuokKwR~+^b(dL)R+P2(G^ zZf>BSsPOy<3}?lav5Etz*5utkk*vtzc^f)&7(m_Vld50}5B0pDeMU+(JA+D;+}XTz zzkxM4;rlTo9YetvLm3b2s+D%`aDWQdEQypW*LJjW?W|S~Q{HQj7@3G`#mx)F-MV5v z!Rdc_eDfWr(K%zkNBDm02!mB+hxM*)7|Hhm6eBwk07kw-Vwspsr>C#;avefCB-u%Bv13J(LPSRW_Yq-LxT z!~px828_Y?!OsoC`b2*X%ivuA!Ol)ohRK<+SIDrKGA!pqwNvzWfJ@0v2f8?Bk}O_F zI?QA;BYcY1hbEkKU2S!oG!dP2FppQGp-BU}#u7*er=5k<#>lBg;D=MpuhEn@WprJL zXV$y2OzjUng>&|-SL5AfW!>zJ^)ayr+ecOI_GPm-UHF^1ft&98B6-48%xxHFsqB5M z{mky}Guh=cCm;KF^}E=A0(O5F_FpU{5);tt(acRwUB=Z(<~RVJ93&Mm7BX+cadbCITPJ{)o(4Blwa;*X9gAeg1z9ERG*P4zt|*PpTN{(i<#si{9oP|nR?V$qIA<{Mss=_B&AJ6 zd*q~~1Y!#rZo;4r4lAqDmUAoS<&Rw2P!T{_u3W|{qa{0x{hOA3E@?5Ud)GoxK-w6SS_<9>4$H8@9HOukYuNKmFHpu;od>YSG!(QVhHV zNS#dFDV#lY2}CQbJUD$qmyJX#hw1jgVoQ<;WJlY*n(DHs3&-#Bmb!|mIAKttrku5I z+!|;QV~0cpm#jQ?b0I(YEe+A->iFSo!NnYrKutVQY{^!S>22l%KST5Lt46LCUIobwT6ex}`$No_ZIGysk3 zY}($3M|v93ZMtn-!{o+EcuqYa&onoq1U-Z);)B+;BPrpee#6OA!&++D8rE$~{ru4h z1OY7y(bb~8{!|nlYdZSf=z8|YbI>|`Md@oy{EvUA^VfUaYQ*E-AoksJ$)20CTPmqZ zYnrX`0PE6}ICUF-$@eoG$ng5&hr!7uo}&8T#_ONm_6==o zht)f0O1qP`OIlovt^>BH%#);h@BQ}$>W4SHxSFh#sp^xa3hcW{n3tGcQw%@J2fdjA z7YaEP8=9TjBvkgJ3z#S)8wwbq$ohH2M6h%N;fZ2x1|83jUL+qQ~vq_KnTK_R`G4y1Y%-lTWcht?Kr85dwt8=Jtp^ZVc~y zOIFOx=Qw)t_1|74)2?oN2n8PlycH{`;de+Y9T=a{1$YR`)bC`y{3p#`fUT{}!Jfwn z%K@&#j@VobUdhaWC^T&A4zjJYc${OHZQa-amgjMv5v}WSY@F+qt!uQz(cNlqmZ|lj zhxKh6SHLi_*Dagf16T5`p4cf^+3y$}p@AhuVVe4k`8Di64xIb;#FoiSb|siSt1J(Z z-g*c?WJz2?mfe+=6b3wckgdx>A=AyAna=(p@~{>$!f8`5Zlj@kPVF2+CX!L4hl0Tl zCln0$dm|DItzqbdB>U*HLE)8KJg6^EOS&LAsZXIK(_lw1EG+Xiafh240wuk&lsj*@ z2HXrrKNOM|!~j!zqS@Xa`Q=`>JJPF`Z}zw$?VaB2Z11KH2o;Qp&@>$Hb24b_qQAV` za`Z;tV@tc|6g!}It`a6DTJy7QlqR1G%ND5CE-TqW0RJ)y3&9-HtbY+{B`k+?qrG{P ztC+A;hG9GOO?BHgeNQg=;m2;SO?_*%B5g-Wq*&@3kGM0#p4InJFI+h1@!Ex*UoY@! zd&%Q}!dO;{*5WS`MZD+80V{rmO~Qg77^0!5p$-0$nNWg?z$9u(^@Zz)EY!M&K)cr< zXav9_WV_E5)~$Tt6!nm`F17g4VRjBDP|EMf`J;@RxU!&ko%$nA2KKW}be8NdI9AVQ z_J3)|{A&N2eD%n#8)pn7#Eq8hY>TvU=%LTJahGxbv@_nL(I($(<>#FDb^22ZZGhDR zO*)7+#<`~{Z<@z!!WxqRML+Tq4O^%4hei^EKa??31pM*h%||~y0WFR^elHs%8926Z z8gRNl0-sM34ry%QtSZfQnP9a{WEn7O^Ah?mWc?R9x8yr4xW7)U$>@WYGf^=z=DjDq zgD_f|${DhNvJV-n1p`pNo}o~UIe3Plri#thzg`$5Z?%FGScsm8S#6kbwd<% zBN6D!&i1io)cj~G`}rFBFFTF?e-ryzAH+mXUG6Y~-C}dQVZ5a94Sq9(ldkdt2_i0o z9G|8h;WTWgp2D?d;2jhgPr;Zis(lndgboL=fum6YQlnjHpFC+CFQCv#kW)+~AlWfS z-WZJxF7wuxZvL`%?6e^K;4H3a9ZnO?PM9gIO{_t&bcPvYn=&HV zE>yBpcm;L>*h|U~d8kMNmmtD`=re+{h^YXGXg1rs1-l)wXEr?_tQpLI)9g0!0Oi{C?|Q&k zlhV%xAxLq(8JI(T&5<#z`O)w-!@ld&5!Sqg@4C&LYkZgR&hfW~?VadXPS{25eQZU- z)S+vR{|#%-p&C+s&1B=dQuaRPHPpL^o`e1<{5jg0Xg!9s=V4ypU{54|zEt0%eK)2K zesAdeX*@u^V)XZ0Z@iyDcld8J5NE$j-iKzbl|o;_8VjPCFKjtLh6`Xngy6{HMvHx@ zD*&6U0qd5`aj|w^2(Dmu={T*GvY9M)_&3GHX`^C? zUSDE%cHaH_`#YH?{I@Wz*bW&(>Eb2&3hUMPg z9`f&I&*nbi0f%2S14HVCSb#v0qge|pWnwC7wcxzceM zbSKpr;xCimWwu>+cIw+8nFan@4!Mypj88P@aJd2R6a|-v$qpp5AE`}QLHB$i-{V5q z#SNUlVP``!CTBeIjAcEVTPLN-_rI^nlUFf~O_M0>?!G|`je0|$+M^{q@Xq=>L9PTx zl`BL)Gtrp#2yE(0*aN3IYJ5t?t;8Hc^YQn^FYtT6P@SlR(gi9?mi8i#1%^ZSBjF;j zT8j7J7yKnIUx^t_IZFX=LJ+~okYFEFN8T)ck+=Qa>Xz-ZqPtj6#l(WriN25Qr)_O0 z+O}IfO7kXu)!>Zl(poEt$vvtj!kVcn6xQQEz+_pa>1PK=C0s7$^E@qGt3@L0Rfw2bdW0dFc6z zbRH7);W#iXO$}9T@iuf9DCC`Yy_r0thh{jHtdi2VNa#hP#z~K$A?v+7y+Gam(e#8a zWwOPDPrm=LPxmRGc|2Ry05ndvXB&HetH<-1WB;GN`t(1}KmQ+3AG~hdxa$sHa>MxX zHw13@;zo34R8`BXb6zZJD@4SINPB)aTFrjgrvYtG*}An$xq7wVL5-Kb?8}|mkZoT zh|oAiHdte&p*kwS#28-yMnXpO5H0^A9%9yjDX`_s4>#dI%&z_&|JfwxY{7rj&(;L5 zQvc5ER|TazgICdCUWE(6gnHmEm*V|{)OFSgbOG{~Frnar>=jTOPF$3FVVU5k^bndQ zm`y^E1+@u2I5v7A-X9^|S~j!F@!Mvbx0IXfgXeGZcy_W6H@Q8}1blwR%6{y2J^YY3 zo0mHB-X+Ig`srUlWBr|QCrkO=`|tgZQ7$*=9iQVi1Kt_B@B#VObp58&`XbP}gr=CF zueNEB*04R%76~F{01X*9)l)AJ%C7*d%p;4_{ zPU=5W)rwQtTBg+JosS+t%;EOzVjpdGyI*`U^(@YX7#%zf2-5Az^T2!KJj!t%q7XU$ zJCSlZr9daZ+W?#wjU&vgthgW#Jyy`9oN&r87){K)0izF5fdXtYL`<@gc{GAnv*nSx zXdQ23?nFP(QdUY>QNNDfC4Bl2P680tuYl8>BDG>q-u&0b&kipMes5sfjK;=jV^Id7Q(`rKM(7kN)aM)c+KzeK>RK*b zeXQv4^YY=d+sCbFS+W$@9j}xH{l;*1RdjO5nM!#``NKuzm_qYj2rmMa1GJ#mP9xkf zSjETCgjLHD=TMdykX%TCDpp0M&^S5j1%%fB%@ZZU^=k28BPgbe^H3JX3YagC&#Q}l zHE*KJUFdW8vd@uzvYo7>?P>tll6p;3W!sU!#CX}b=K0ae``&tg^Pq6nd(dl6&Bf}A z>Vl5l%ZVqm5M!c!g%K4WNUY7o^yxS=12u`AVaAD-5PfDqDNz!Qs&AYTAV$!18&E0d zWDctXVBnOEmXt7|WLyaxK9pCW&ZHn8Ae6x^hgp~$zR|ag!RmaYDS1_lY-xi*v>7YOnPO@2Rfe&`lq?LQUp${1qX@OEmKWTZsk%T1 zO6cbgyU(~3O-8{BlgdQEfXghw`4&#NzgIMC(}M7%_UpKNn` zwtroH5>hmyHj}bM($nhK)XrtQyV}Yl?afPL&3|N7HzA+St(P3z(F?DCVI}sLjU15| z`{VdRU}H$H>vx=2kuiSI$cJ?SLyVc8XoX@=J#SbBb=63Uqhl~11(ZDN7S=-Denvm* zJT+H4N@WlG9beN zC6$zVi2We!ya8xeX0zMU??4SQ%~K~&Y;CHK&#sw0vM3~A+KFi6*Wg6(aC8ZAqWsWR z)${9Pj|K*>#M7Xb$Gwf@q(}WKzYUki)>ZfdGjgwbCz&L=?4zr0SN}T7}#$#$nl8z7|M8`B45(oX__3`@J znvrRDVj05yzce*s;&_7*rD?h{5#M=X*E6Gt(h=yfd--QN;6S@aG~cA1&<5cmACzHh zO(d9?39a8!5ktWqrvm&j1?536%EbWPgGn9e#;G;Y@ujb6sQVJ=FCU92^$T~_(mpPQZbZnl57`Y2n!Lyc^-XUYFi ze=AvCTh+6+v+e4U&qldy3Ueyzk64WbmDU{{nUdPA+A=#jZ1^=AeBUc)P)VmpW-qa~ z<>d9#^sxLl&?^S5gGp9Yuj<8&A!KZ*s8NB8Rgj05FyIElw$CC`m_{Fpii#+pd-2Iw zg9|8{3L9EXxKU~_f;|CO!xe5ok#dOaBg`8P!6k%Yk=1#Bjjy&M&+k4<`XStnYPYA% z(>VP@eC48jKyvbSK)n=8gLLakPcW!F& zqB@VZOtY+p|5cXuSIA;i!3l@gWzF`AHpOb&ki`_A+lA?=oJbyxNJ3_2$-E3Z!uLcg zSlbMsg}lW0l+OuDmPzRlva}Cp5rt3m;jTn2yhyZ1Y2MYmYx}m18`cjjTQY0rwDu_g zQr0)s!}3fXMYbPD;Xn1GpaL0bSe6aeAr6%P3v<`&QKCTQY5&5!=Nt|l3U`u{tKO~$ z<8+!i?lYx3;5NqHo^p5hl+`%RIGyHi#F+YRoTf7_UM+flvo7lPtYdHXyFK$}L~}#7 ztft1|uK486Y2S{=oEA@3?u2mVA|I13QorARWqY_Q+D}YSbk5@FU1MZ|+!nU8w^wze zhTSi11>2zPjMzrq(9{y{ENwZvwlFxUY*PKqvYz_FN%^%4d`e}ZdX3yMZq@fz)SA>Q z{V0GZK0@9d%ybX#NtrM+F)c3#(VTpb8Iuvwwo$-E(IBh?xF&>P#khpETHu|!KWNZ@ zU<-RzR{)XUgy7pu>y z5A82AYSdDk!7lcs3?P0-hgIEi?DwXQCib*?h)_A$4tyoPA0Ag9hwag<+WXS|-{cyK zt7~gM9cc~b41q4j8q66g8W!`UIYX&y7UDUnkx2eNsKHNNvt;NR$SUIusMj_!M1?`L z_{%{{L@5MsHUbza+*lx{U>pFkjtVN#Y#YEe6rB#yDq7cSaS^oeCFw5Xa%vr~V^IPU z7UcVJ8R7Lg;384&eI{z}saTZ_&-{#bO6+H9#Yscjg|YJ*W5fLehZwTs=5Shpxv5zE>#e%X?Z z^OKTSOK^Z+frE#fgC407x)P_QXp-t;=nVqgN{h z+t3F^DDg{@*&2f_&f6XfP@h&Lp3I~|(WDh+_+&xh7DK5JQVJU-kSp?iVO&xDL>VK; zl^JcOv@mMY`9Nk$v%a^U&vvQn4zPCCxz7tg=g&B*V~tv|G1jhbl^x1z7|U0Rez~Jj zox@Hal+39$-VdttWE(7LybJi0_H+yQv`xTd$Z#<%W@JVbskNR-W`eUy7!E0+`HRU4 zN6TcUZUQCqL`Hi;%u(A?n*b*R@XqRT*aa9dprz&0F)SxC!ZaIrU7(vqPL--Ef`FH)G^yDnS;zr~ zuMczxtR#~F9F`Tcbb+3+D3(4X088ejhW@#ol#mmxRrKM~vNX4EjlUSrn*;P-0H>VS&%7Ho;%KRP#4eFRx zn%F8@tkcHB9X2BdB_R$ifmD#pW=Ljqa?+#}S3N^+^>ktS0&$d_KWFM zqN2Nc@|cZ(4PB%=S(HN2bP}?x#ucB^55K9Oy@@Lxo)+BhbSr^UyQ{f5+!>$T6)F05 zJf4YrUOzY7;APbpt3Ou1*j_j*IMPMOxL4Mx{;S1fD_t6G-Ca|VKdG#+HD_jde_j5> zyxtn6B%to28$a_yHqmZngZ@xDk3mZlkKyu4LJb5UvDw%JAdHbDZ;=2*3?Z+Al{TBm z7)htqn1rBq1En9Rv&-1iM;D(mSBYQ0tMksIpNw|>PQ+Hm^@CnOje~p>=P7iG3KhUk zZ)pyoAIl^!s-I0FYVFWX63yKyz$+s7#B)#mnvmlWC@2aZfU2uOg;H@*VSW(BC?;V# zn@-6n#zT-=zKO)X9$P|wJuL&r6FcB6gQw861#HE~8<@qv#=8AkEj5RB-u~qwk9(b3 z)laY&jjLAdFAaqY8|p$`@hP1pMc=BcC@d^kUUB(fZe0D4`lL;sZN6~ny5}C5qaL#R zL~0Z!`s~t5yI*S>=C`XG6ZP@6>w2pS@>(PLGje8D_HJxx?sUz*|Jh9)H&a_6(w(q_ zNOw-6y7Jl@bkdT>!*k1Y038I4fcu2(h=4Q@oH-24gf2iWZc-JbPzg^18g$|Wu0JsY z(@yi$$*4){M2bNUBTd?kOO0ya!nx$uovU^As8)`diD2)fD#ue~B5o-0ULxjx(4|}G zO?0=l7G-53%0JC4+QiizOXneCCC!D3bfAkgCN6_QK210~op4sP?a7bRdJ=DG<{wDyk6X|Be<1V!8>U zk=v{=Z|*_HJWDEy+X7i$iz8P3t?8YUatwp`WNcKANP{JW|ym;i?6iM&te}s zXQ?l%ABQ_5{c6>^Xh&H6Lx;?^<;NBO!s`5qg>`GnXVxXkCIt)sZ}Q#)Jg%!+8$SD- zGfks5nyRjmrl|L3%aX@cmL=J8Q*o2kEV;@p?nUk)j#F(XIN-!ib*d9^5&{H563FES zNJxW_kPj{aa)ApW5Fn9G{&($jW;AM!;okpw?)U$>xxtq9oH={%wbxpEt#`evXvgH5 zeT&#_;US*Z{gst_^GTan{Uf5O?Zs zpPy^7+U-_rEQ||8p3A)I1`h85{uJC|+GGQ8O}qn)^Pk>gi{9i}wN&|or)79Ml<#U= zJSibPnszwd6Wrq1ik?NmEne7FX|a@*k%by(iyvKlN7p2_zfg;}h3!Xhw-**vSJ7r$ zrl)GVJ!xO-c)L9&oX2-Ncn2ZQ;;-Qj;^I8Z0G|VpWky6di2yxmI|ijQa@6&7h0{11d+Nm}(Gy(wW7Mf|WQ01tw?# zByH4g8&dq>?kK|#6zUTLS>*4_ed~pqZ=F5Tw+@ccbwU>Tn204}l5#zIfm2V!>$RSP z>T@WLr#|OHd7S5(I?_8f3XWsg6GV_)$; zz@zdp=#i4j-)sCHlI25x9r4^nqo0d22SlLsH1;=DSfg&N7coE-oKRmlNZ|^WQ&b}-2_^V%PFZ3GMO4C#IB_r|DDo}{-4#!g^28!q-EJ^_2dCd)e}jM65u~S6)v!C0R&QO+a)%=Z%Io-n^1gPG2^}WLl#?t0UqnnC zY8Fx~BKT7XgQTrUJ}T>RDZ`2Y>i{qxE@S|~00Ihf56nDLh#a-a1@r;YOzUw~pHK=frNUoJ)XLi6ntW+}mc1Y`ccyMx;@%Aj7Ke2` zd#ek%?);?uY4VE1z3U_G5vq!VoP(a`b-ig)dfRawGlwnH!#Y!yyX`p6#PZNup(cQz z;S|`-C_=0dX36+Js;j}jZUvDr7}d2z{;FMEEX)(L}2NHlfk0 zpel?U^%J^*tNBPEH*B;?2ekKB)I&vL*6K6pa{~`OVopUP}`4RQXbLQpu4NjSpT3oPREeRez48fH> zC>OTWRL=FqDSx{d6PcejxpZYqv_GXHy9P@Qol0^=@&9Dp4g!^Gu zsK+3QT@wBX^*6HLh18B-U78=pe?6+-Mez3_KMc`fj@C%ZQN3UkafNqmq}ww4MOUV> zH!@L7StlI^s@7^%_v3h!@ds==_sN2#PTUUzHfsvjocm8yV;~VJoQR%E9^TbXo0uPg2$m{uV#U%lBEW`5{BCZ&l~1I-Dysic=6sx%kv|EuUa#dw|q{)XLc8#e{cD8Yyz z1xH!I+t5v&b^~MFp|@@JDfXSMlV)VFATYohgj%@ZgMT24Io$0q~%^JK-2z zPSEI4o*LaAHNq$DQiod|#f2=vRSPhTR*-)K>eJN@|7?C3BKDx1fw zbK2tDmE*1Lnbmsn#t7^6M`dYvCBqFw__D{R?LD-N>#G@%p<5v7Y#zBEXa}Esf$7;8-(#=%YqS3n_g30P8=8o)a4Vv`Xm53iCWGe)Ui4O_7bLk zq^ee%n2!6y5z0*E1mFcfegJXBHN8)6vf8$=pRBZ6@7z;)a~@FYTiH)nqL0kpg*)r4 z>R+thwaSx?8@g+j-%_yh{)8pxw}@XH)^9b}^HFHR}|6hNuAh$Kj@3^e%z|w?>y4{NCXSUyljYh1)i0G9J&!8S44I?{-%|^T( zyg#^f2}aPq5OQ5Z-|11XsV~rTq9?#JuBU z@+@KlYAPrR2&Kr=SR#^|L9LiBq!S{}i~plLn8qfkO=g6!k}4$gRk=13byKWg>;+d1 zyRO%2y9P@v0Pl)& zg9hBd3<3~|RG8a?R79}m7UB60$`7)6=#T1k`uwp}aI3LOQt1n^N%LzWxyRu>_Ut?h zLHyGYUtP9R*pYdDU+cXyyB^upxmazpXSH0z7WX`}w(qGe&bY3Y2%TL#ux?*%?%)nE zv#RAX)Msy&F$S2nW3@{gzqoCvl>{8~X#+>CguU8Tq^W_EJIGBpa&-7W_-zIRiVSQZn6wEWOHG;rP9|ie zBVm)6;6m@g7?cqCncrkYpA%kMfPx!dLZms=V>eqLwV=+EssD0Lv(2`FZ4_5qhmZKv z%JF%b-m;Wi#ZA4k8gCeHpJzUE7fdhg$z=2_xfA73eI(PZrF8gZbitEK;lw*(zYXn9k#;wy@x@}-^NKp~Qa#T*zpdeni^#sm*k_*h9hP#l)%CCATZMN$xg zNC%1q4-}TGOtXxm{M`|%6mZ>=rn-zw!! z#eTQTzv99gR<>n-ZSE&dNs}0M4VRdiE1K@yr5xS!!dCcurKsb}g1n?7@`K(`=XB~D2Ab%&(HgG z#FIYc=N&xQxMRX-A?%1?C9YSJ0HS;362Q(OVY(s=XwYfpbi6(_!w_ltykxO?7dxAS_zYFn89B>eiap;wi=8#=F2LmjF4usi6n8091BstEYT4w6%M5|D)lg}t0 zUB0aR@wQVe=IN(c^yxR6A3k{OvBgUsK791iC1L@wQk-R`2kvI3b9aKJ;>b@A9{LFg zDvlm`8~XEq#d~0#_;dT7H+$6ut&w?0s7DkN`-eYQ?Au%ZD<2>iw=Re1EAvk8t zbLeiVzR&%^F{9sy^BQ4{;K!X1?~JpKcjvrD+Pjb7HR5xClM#IG1A=<@9W#eZ&y_}q)?cMrcW?!=9s^W3$e_kEBxs{1l}t&G0W&yDE&-1l7^dhSEQCu5&WcV*6z z!Pgge<#&YytgpHkv5#YY6QC!tzQg)bm67z&b03{(-}J*{UY>Bw@cw@|;TVc7a2cU> z9A9RK_fj2mRWi~LT+iTVp!+{>{~sFjnQ5axBMx(H@wMW%g}*;Ne}eb_Y4Ew~81?sF9m;apCZ(;{H$pEbSdmJwIiN; zEBiB#MUDJ!j2Y*->evy_<tKdL8wka053L;1UFGCDYR_pd9pN`EXxJ_ogFgKrJlowpf?N zg$R8Ni{YJqie;jK3RzMoSXvG4e31bjOIFkiCVHe(p z5!^B8U-$GiPyBjt_SWZj?S7t`DR9{h?T3$8knvLvmcd)AL{C7TtKN!k9=-4AbsyO# zT3vsl;i9)>SbHMA*Og`19OA>_?hD+RXZAm^%)nP$`3J52y*u|`Y@;hcLYO1NSAH#) z8H+8(VcEAlH$c?P61}@j{U~E_D=ZTh1)B1FqRx#(uph>+DNAzP~T4G$-DX z0at{t!2z7@p97GQf#4F#v{9{sdU+JGNn$)gQ&I!( z1e`6e$CK~P_vU2k&^J}*=M7ThJaqL3K!vD>=s4l>%P3Mr{-_R{OE)ft*+;is)3=p8 z(uR}OeaqKx-a}sMqT8o2_Sc0w;hh$;#qdrmixpk=#)Czd8b7+3{L|FrJB!ZG`{Xno zDtG$R;-4qMRbBbm>qp)m=zDhZkTK1t>KDw7`-+!G_=+Jv0P|k}E}O4%8AwH*yf}bF zjvL#Lh`h$agfaLRWM$-XqjpvGxjk^ZeHki|r^9uNxfw{SAp=6}T$1?QPB$rWlg)R&d;D4DuP4OC%a7jKdT}?+?UERM9}^m44SU$(-G)zUS@U=sPUueR+KdU<>c#sjw3uHTF5gf!+=tAUJr0(%A zB>>b@DWlzl&*95PlPBH|F(iV)yOD!~U@{7W_e2g!RwYxtDLuP*U`Bqirrd8o4dML4 z?-tJ8tiLBIc}3OAEy_0R(lPAPqlb?^vP6EgW@ksr%m-G6EBNjBk5wDPpd*}I0U z;WsIgOFkbimtPwpmzs~nZQDvqeV9&wXo`qNSp zTqt|sabzkF0Z9{A6(#vY&tdh41~S`}H}~U%|W= z0v8}*-g!bb`=v?-8(~J6GzSPlFo>(X;@VFl=_djDOH>ZHzr=<%P;pS7tyze=BN&1{ zTLzOUf&gVkAnNGw86t=i%`BMAtud(nk4E}cSREBK@u26xA65EK;D75c#o}SJ!yaac zK;_B5{4`>=tyQ0<2alP-^xu0N5$(_U+{c;tZ2MwZBqkdv^h`9_;5@JJW}_b^L`A78 zpb^R-xN2?qC*Uv02i0m>esw`LiDE{&8{PlN`?FezEEEPCY;#)gKa#p~vR+_3%5WeW#D8o^k&~58=$~h|u3Tw#n31o4 z4TVT)U4!2bJhAEM4doRl4<5U*to&riS(#e7T>125f^toaVW%hZl(YKvkjqT;y*d~$=Ruj9GT@$2X32$($TS1RAfjy@XaoUR1VXb51sYNZ zH?nmmE)&Q@eMRo1rV zE&vbj;If`AGrH=JJ-WF6S%i|Q3_-pPT^@$kH(k5%UKV+lj&5R%Io9p@!;$LEgT+c@v;gU9Y|Xt+nNIeKqHg}hAJg3Cm&d_?`Oxke^&?o&`&V;RLs4CtY=arTsv+(>9T~PZ`FDZ z2NXRXPhNgT(0W1{Q#iG}A_yi`xg=4aCj}KU?nTzHKTDq+eQWRGw+_r7sHLaja2N)U0z@k7=0 zi0Pqu&D&mGl37QSX-F=w*)m6+A!feqMn1WOuvyji)jHO>8YW{Zi>jL}>zwtyGqgD_ z#r)S?->J@zWVH%+l6U02f))OFfslp6iCi3ro7MzFzR19#exF=#%+2HV`7=FwBY8UH zkI^QPhnXUd4*z=5Rg5Phq1ohjmS1>l&z`sLUcUV9pX}ND*7;@k_I~Tgk>~pQo;!Tx zTYbvwvnwlS)BlUbd)_?1bm{rm_t5_?yYTv+121jc`24{BAMD)v(!t6(b1EzA>hLDm z7vOND-yuJfgc~_dm|u^(A20?;#S&&*67&)wr@`7vEpc$lRQ1f@Yzl%eJvl2S3m;9g zm$`jF7Y%d%b9fPoxwVE_suIB!R^1Y$s~mw&8~)kIzQ*)_KlHO>>CI-Fb@!xO9=`2y zYlY2O6ON6#{p)-0Y0@+04W6=RpADSYc8Dn^S@%Ej{+aato`357(emG)qKinV`50^7>Zxov5qbUA{VAAZt&ENJHf(q)dzBT%dYxB@-5S`3idfOV)e?W3z&RWyY_EvMw#*ru44k2 z+y~Gx#1D;%b)Y&}VF9QQQkl+}22?v|+%Xwt6Zd4`7VCzcA6SWy&hh)8JadR98jtdN zy)!Kw=A9hsG@^nU3E^#b6hmPK)eqf1knRquAAWpSTH@m^E0G(HjZd7DIQaWLZ=Xz< zpBNvzF7xy)`|eJfl4xvz*J6o~HkxBqGSyh~687aMt~tL*FZM-P=pvFnKyTgKI207X2 z@m-Kzo>NY6b`SbYl!dBA2&0oh7kbpz^6M6%veAw%IL&di?04Ja68yD!1<`TY*3>C! znblvqnXO>@za4!0Sb7T@3hkME%R^@_Su3qv<5~?;e`0KG($vJ1_}MwD+9r@#fbT=+ z@n@K?2f5%fLgN-NqbUWON|fK2a0Hr4$`R&XOqcI2I3nrN!*@55ySCxGTU_kJ0WU5q z1`kMnwztSvga>>0E_+c;KD^5u|8-TH%n#{Giz`drXRxCF{nxwRI1%5FfPK!n`KEp6 zl5+jVxk{4=Y$M<61m32Z#hRaGH{HGNRJ~rRQn%W$WpCd#*y^cFC$_Z{$$+Yl)cI#( z{vCmpCB z(+jHI&s~6G_&%ZSSEZaZZ^-x2Bz^n^P@={5+i4MW^AnA8M=wB$c;h|mPBrMI7x)6) z-QNCRTlTI|mq6QpAUt%<`cA|krwDV1ODj7|6b)3GWmAogK{Ol%cm=>~a4wJ^VdfSG z4aGxeS`cbyqT0NwFdtQBR9)t9d%<3qs1?_!I7Sk2xJlYoBp-C*aBS6BWROw-XTZQ6 zb*Hnq2Hq5Tei`hMRoM<(ExVQJ{@(S_ySuHn-O9(SzV~2X`^g!#8$H%bMSYW39N0L! za^swLzq)3|Edx#DeON;{9$oA0U*B-kq73Cd)$=$fX0)B(zxvkonK`XR$=PCca#hXD z^&6J-75J78w4c5wLXIu#Z-;ZDo(}``!Lv9YiSP)g2=@kDI%t5&%w+J{Mah_#E9xZ& z(cEu^IJG2m84AsSkPsHpUC{OEW&ya@s z*)i|*M5}f8(5*9OeDU_Oz8ur6u7>IrHzDC`y5*tUE@7daC84F(w{T^HW?gUbwZ3cV zjLzE~k=uNZ%9&l+rL0|b`hbU3*R~9@_Y7a#Ore^rN)eJhVrjMykzxL^TRZ6xVNY=+N6syuVjaJf^@WZi{f7sY0-~fjj=wjUdmz5j^W^qLftle z;WFhc>RtArb}x(2WGMU+vr=46UO0vFXYdokPC@mkL<6)GxSXUlBIYM`z)3{R4=4vO zb>PX#h_j(W4OQgi#jVy!Rr4=QR?ar*;*9dEwNsBx?Ob}I{@|}(dU%fVo-t0>Bu4eV_4bC` z&RJV#ah=+PeSHA?S}x2K77FVEeRklj!RBigo90(dhBYZf#3Z5!C^;Pwm4IkELNu?5 zjf{~Za6N&?$4N_Ya7rW^W@J=D zC0h;C-ILaxUUBTn752z!!+j5?M>tB zjMBh3zeZteN-{FDJm=uFM@9tuzc8o?Ee6aZEXc&FIpUmKenXhlJe%61NPfLK7S93IQJ#xulL^i z=krO;t763#gPrX@_V$5+w~r!fbl~0N5FyITtMf&E`b{oV^Oc7~^Dh+E1fmKHte7<# zLa1@K{}l@X{?iE-Bjwf+i=itJOA9Al5wm6FiYUt)St1oj!e41pUe?<+Ihcy*qa8$o zx_E-^OZf>X_m>*-m{6w*MmlA%5osdI3dqMMt@+PL`N#+#1Ton^&M1^DkeE!a94RQ2 zJpQKyrOXR$;}1dvZQ%d9{Oo<2=zN1&&m8#m7Z2?VEOVyc7+S4Im$W?guz2|F1Jk}> zY_4gneqT|>U$-dT7)et0F8W^K+$jm>_=L$d3ntG$55+`hsR{V?-^=@j6pCa8%F$e0 zOiM*x3(7~ANEMb1aB-;t%o;2>{-yku*a127AitGq_j{o%lX7%wR0pAG75c}si%!z2 z6=?dxhBXSswsgPWe%*=33%6H&NBPCMhI4W z|6lT3E}dPUnRSp!&#!;=Voz53r{|VlchXtN)=D$uJ!^mV;Qeo`&Zjv{N37~HbS!x# zsG0<^md*pvF~GPYg9L!3oFdAUHm6C55*Rl`tqT!#tqQjBDk@~cB=RIC=jNm&C4v|L z<#RFU-BZ8{sBDx94*dC0mQ=SBEZ_*%A*pUFVs%6Wr78yQN2F71NG|H#RAIAL?%d|c zLQ~xirEC={(pIrwy!Pk0yXzJ&@tGt2J$*?thfdsa&zZ;UIrgNYgod5d0@pM!QMpMg z>_GXP%6oJ4{>2-n#km&sl$0*7TxyCGd%6y+-k>xx23^6|g2!Kero~@cN5)n##z*TF zHs%8AHVDTXj_FgMla3gT`kY6DW5Td&>KMYDsbfBza16zI_!z``zBcB4rh^k3`xhJo zo}Aam;ChG?JA6zNzKiOH!?8bn49EW9yMQU;V>srC<~Kh6ZaR3h!=Jlo^m8F!F<7s! zLcU6cX{ea8CNaGs3&mZ;rztfQ{g zLI+4&C~BG*$Qu5HDw(4{7aH3Zz#VK_^`x?*!VGs>3RN|$@@X~*rK~hc3np{X0owDil~C>@1`5%s?9qq(T`yJRy&+tYqwIo%4U1!8Isdqaygy9Br_^HtGmZj zIh5nAP*thCH&qf@{ z&py)5tOS;q0QH_oM8T0R6UR0-2FDf-(bXQ?vQnK~J*hNMR#R9&+<(-z)@F@1%O$!J zI<{0XDE-Sv7u?N{*pK{^$2Yl9JktAe@8FrMJ;cVS;X|zI<*BGeBb`R8gF2Bo^wLX2 zxqE&dvP4n?06?;U=HPn7CG@z5qeKnswI)2ET~^8jby+gKgmeNV-YAr?65h&(Ttbzu zXtYSc6%1E$0FB@#8jLhSu!X|q+wCu)&jmCn! zl49wFF35?}NS7l~&>?s9@38YJ$Q*&Q+zJmm4#zB0LR+6Q2Fi7CL&AZ^MH8_aQ9$i5 z=L5no2p*)~1;lwUYv`7q4NAY%inI!+9hK3tkRqn23*z&a4TDEW#e<-P@)wvrnii;Q zg1dvt0#{sIK`Op9ZDxkfXFvY$TUu{vIeM{r(Qi7NAHJdaCiEff*>gijQSvz5tg>vHAf z>M#A}^>xkLE6WZPOnUs(>i3k79GIx(LE^ z4;H+I_L>7bRG|-MvA7uzn*3Vu99S)CS<$Q;cGek9Y2L3U&FHS&cWSN^N!(ER5mLm^ zgz{U#`dd%iU3&}9-cizeB6@>qc3w%*^li5;aIW3gXQP6oPw(L=W6XD^a#os+`MT-G zn}9uz0_&j-7Kd&##QLqrG!Y>YVL~7YT`()^osjBubsXwqn3ry)XCk;nhWlugk-j#83s?pvm5g3&Mzo{T;)NO(V7&cGPZxxA4vob! zxV`FZD7`;+(@@r@PC|3ZBt#2q$D2tB=JO>x(%RgjjT9|fO8Pkswl zda80%nk65?8ARPzMglB9E&(jk45AtoAwl^_Ds6td}Y=uHnZ|!+8gO)Mnc`(!NZd` zJ2skT=a&|({n{?+-xE;t`}m4}hZpY)QTgm6c`NKkKG1|;RQu>g!?`e`>K81UP#!=C zh5Z1rg$96xsNHTFIm-r==&eC;3b9`x^w290u?IC(DnWyILqq6tpaYXXViF4 ziVFaDDJiHbuF6DLT^K3 z_L061>o`YfW-qA=6xWDj8I4Ou9=H-|Ezt!@ZFMy2HvnK^mBW5$!f^T_;jyBy)RmAM z4UdIKZLvg)I^+}p$yyVHWC6X!%(1iut7FDOuqK6l-T*qrf9e6W*OS!;=m~WNNBN(8 z0(EzqdKFIq(u$c|X(|9IkDf^&5cUL){2FH#K%)^htqE)WmrBgjQ4p&S7)nf?`Kk(QwIAK~@m1Q8+7HVpZD)s1>&v zTm&gP&x(u5v!)alBywBo6icMNVPTFX%C~6#HK%{|2kGR-WMHpO#zr%HwAH?1h20*h z{LUQ3zF|grVCUeB_Skq=p{uXI%$hRo8uo{TlK9x@nrDulAXyc2SJaiSX?7&<+VG_t z365noD0`yyyOe{w3^B@1y@f-vAjZ*A?0D^_n$>$M#7*p&(Xz39`*NrNEOaCMrUS4g z`2vs-Nz|c?2Z*4JQB)0O#SkM4#W;Gx3?>sY+$qTkt_WytI;vV1rO#W6I$Ja6%_xdUn0rlY_o;;*yYi=d%!O;3W_M+m-?nGTmwWy{H5b?xzAya# z{UN4C^>-!wP8?oG%f}cl;ZBS(f?pj27v~*zir2nrV{!i-u8&fmOZRd0x%W@>TwbH5 zj`?t+F=5}$$6VES(>*ioWOMpc?^!h{oFlcA#O1d?CR3o z(WgdpFEt_p>=7>XK5L%O>L(Z!Skb}dttFdEa4PVxdex@e)rf-=es<=Fzi1mu7Jt?K zlSemvCF>fOXpKl-K7ISy1-EwAA8Ok4-0qGCs@Ehjt1+s6ZvVj1y$?KAjc$#ULD?o<``%&0*^jwGQXC$6j(aqDzi2S)zlAiq&Za?$WgL`z)Utl`Fl`O*$MNF<@=3i5cp&_vBlh`RkRFSF<3>&HZ&Xn{#44QFF7(R<70;<>pOe zORjQ#R2^hkg8f(Ziy`VmiE;}}m{c5`p|5s&)Kcm%|zt6GahI&1R^eO`aryg+S?smD>!SdAmVYg+=Q9oUsS?LYU3fey!Dr2$Tc1w1}G#}Va&LITOGnzYWy*Pft%t` z1!#+<$T{esh-OfAJ{WTs;}R`>GT6*cDz{1P8xxIArI0OEcA`m}A$4Ob`vd#da^<&Q zOjc@@)Rgbg{3wQh2zqOe5G4etM;1^kdh{d(juT)reH;IRytX=TE9AD$L7Cs!&hcB*^s zhMs-g@{@Qfjl6;sDZ(rkZ2N8tGE`m zJpT{+pOCa^A5E4 z{x~gRo$QqAF4zZmX}4-(vp;UXz?^jF5j6wW7e9WTp#?W4T-n^8m z5zn~+&lzo>Gd??jF+YdkrV5>b=tOKHh|saCK%&k4pW0cE#ZsCMp5PSKhKF_+CY-Y) z;zp|NZ@sj2+2D23ZNXhmo{}8C&rQ1gqECJ-eV^~P1ule0JW`h6euv3uhOjF@5G%|E zNk>gaeRQFQjsnLyXmcpN3zQWF_;g($emK};5NU~$hJTn>P^y8l0fKg zDYaPAW2qsWCcZ?#&%O8INkS=&5@aIdw^>8e9Kzp+MOxY0rL01#Dl|``>OaCon^cTV zJCAPRu0S;AN&W07Is{{^vRtR5Vhc7HiF-_Y6HcHS(4&(X#1cXaQx#K2_();@_(KC8 zbri*7@W2U%K^MnD1G)vbE#2brdJq~+)_@Z@-+DtBEK$|aYA}tJ;yAySJK1zNdcHqv z;qgAHLKOn5ZM(L|lL|VLOJ+8#8BjjZ1b{lpLN6h%kMjahN=E=KqD1Bb7?xMd(A45- z4e+{KFe%~4h`9$6wBPY)=g42S1wad}mJpdEjCvuNC6oDOJj}0c;_&;cZi+OI!uZ+- zs&5}(^3}xRg<7coE3|d=@}K@X6qX+$v*!ab$>65~1WeFJlb*yzQc%Mi493G}5XAi< z)tWopCoP{_RH7R6se^&|y7FjjIw@s>hN%O`Xy#l->}O^-SIsDiHG(=yvr!ck!X23t z(-s9a2$O9^d3go8&5iAMjZoB5Me}viEve1ZCnX*=I`YSA>=oQ}dXYab!G2C-34!Qp zRNSR7DcVhUZS5S-0gIQ(Xx<{2qKu|ZLNrha(Gna_DyZL&8DRw>qfvq!9VMf{UWa+= z^t}*T{JT05bjA_wLPP{Q+Ccg(Sh2eB=SA)R?>sOtSA7<0_vq#RTMxt9yQbDmshV7g z`~bLz$w1nCK6?hW*91I(oCOW+;8fj(Aa2IsXI%FTYaVnIg!!~{)oVv5jTBK=GVs%5 z`yOhktc>tQRAtQ1YZ;i;vcD?cW0=I&DpfY?4#m9P1`hZXF=4Ij+>oG4RQ@hrqujso ziM7(|2UpiD%ks|D8P8grt%n0mJEleI%2?;nA4VJDza4bEwMV%qci(}%MWH!zlyhN+ z9I$#hflR^^0N@Bwha|$W!V#sb0)EH~mc@A%YkoRWAbDxF95IA~o?ulnXyD7-^4y|t zUVq2?x6OU?v1!NGZGK_TrmvI^!i}7oRK5JvrNwVCd#ATU`SjN9FK(|+lf1|98u*TM z=*~&pmuW+11jxT8TI9yy(k6fao;vXux^XB%FOwKZ=D)_<4vhEvQErSCm(&t>FdxE6 z#knsE8$hecj8@wY_Sfy8>5rf_{`3RpykL`#4X@zfZrcUr1OC3;N8#`Le~IsfHTI&m zyA1E_74}??r*w<9#$M98Fsx`lVPXb7qT+%c23Jp&_+geQ;M2xYlH&>k+XC=fh^8<( zk$Q?n5iK{`BMM&TRSWu%2XL$MA5z#9H-HAyfO~d(tmM1+&`445ST?tvh);jKDl_4> zd&KvGx_hW}e`ouSpPGQ~Z*A2}u3suI!h7;So{nLzcKoT3=`I86BhY+FdlyEarH|ey zZJ|lvn!E)&ilTho8fvknAsWntXy8{PaDPB8ks>b|^*K`1pZh_gkR0QLd;rRVzoC9Wd&S|e8ut@acbn$(Ivs+5Dj<~IdJBC8sYwoxGaV~nx6CSuGREgH zOCeoMDW8eMn+|{|r3W>5A6x-_k)_bV!gfY1$e$YjBPK5Y$u~CseB)-DmBk-Ekh$kJ zYh}(9r8Q9>-y^M9?KC9FZsj&v{>E3XIb93az!@Q37_6SOq2a_)SyoJv^yQQ@%%%MH zObYak58ROG1;r~QQR7W@%;>!n0~tYrC4oiAzBuGrD04IB#}a<56?tc(eR~AbQK;{R z4<(>cl8AJikr}(pdbGV3+fBIpifO!#o4z6|IG)m1SN=dCn~#;`2niMpqa^_*Yy78U z)su5`1tGU0x1y}L5PTMxR}Pv@_CT=O|qDvJbLK@ z$DK;&6SJF}vl9nYeaiflVCs@%ox2u4(PxY2f})~T!P?AaRJ(cFLhecw>d5Yl>@3j z8oG1@wSp6HWvm}sQM0n`{t<4Re|!5aUu4%IE9aE14QhhHoi32D(`uyTD&07n(0Nl~ z-~NU<`h?0rnG0Mws9;4CZ64V{CuS70i0$xB2v!cIMd`Z)TO8p?&}GNRd_Cp|Rg(W@Bq@`PiCzw5&Z$AE`JeIM{}%pZS!h~JII2!3^px|S2> z75E$oSEoJ~TwN2am*Hgax#}3C6sO7WaSpXmzCs-4T~pTveId2yV(j>PHT=HxJdLyP zJ@tLn{TRVn$e%mTXRyzo^O<);?;rdgtn=r51{`bZS_a>Z=6>bR=-&>`S$LOyYy3Iu z!Ih$|OJ8tZf^r4@RQuWc`mORo+^egE^#M~^8txU~K7kcagiiOOfJQP%J#Z~V8E2&f zfm%~Dkt0wz==zaz~#-s=e7@#l-*eDe8 z!Ab^QdY!7N^%+-I2W;Pchk9BIlICXUqh?mj*(!g$&T9MUw@P(LH(aaghBfSFX{i0M zf2w<7raw}jVol8sWa*hJa_HC(o?|zNyCYppKW7gA$>W`;7PXvNF3#sa<9Y(|FX*-4 z`XT=H-kA8;ROQ5oF`r_MN5{6NDqjd66A=iG5qS=Z<3y$+Mi%y6$Z3uDoP#6A+$29S zGPXrJNvRz%=2rHXk?(^&FM+-ug1+`6X9FCZ-2iW&*ds&{ecxFmXp};1Ce&AmkU_-K zhH{atOmxJ_%Ph_+_8+T;tw;xJ>=T0bp zvQ45e?X$h|r}8f_*BnCJ#1kPnYsg_DI#r`N&O}xU08qHrc54*(t7%%?N}+<|YRr@l zMGN8$@4sgS$?}_PR8#c@AcxknTfngVm!W?wkv6m@{@aV4xQE zAna$SfAHLz0$q6l&4md8+%0~kdqk+DOk0`9C4nLeW(i0r(FFD#8TSd~QpQ-^f;_)( zl+~?dmHe7;F=rQBjUv&)GGX9)#ZJ9cT z^duhxPN}hWDF99>tbJ(AnDdGKLC#Bl80U)fTzJ!?DHDo}7wS8~vPUJossof3?~IM1 z)RvAMAFZkit+ILBPnA@K`#z|c@)M2;4D*B^jTXhd*S&JGHqhdcV-K>r(mHR)xuwg_ zb-L$NvHByiI!{}m_J&szmVfDmzK&;3=g3B*9MQJo&c}CLe51Sjjf;CP-qGC_0hivG zbNZQ%z8Ag(bO#fr;GN$@&L>mI59Clh6>kiWoC-9^Z9`N7g>Xn_aYUdz&+GJ9{2G}G z@x*cbFZbIJ(NtYFdhMV>PZ8xek0^h9EpW@h1#z}MR@Zv-k~QDjSzPCS&YHQptp4cg z$>QqAly@J^YP+#lyq?-fFCKEYADFuHXQx_@u85B+ORJjJdDk;Bk69g`5?{ z9yD44`fUrn24X$3T-2z(5k8ySh+v#9Uq_c*H^!zTS%miDz!~F>`R}pL+J}N85vk;Z zbOToyh09SiKA$F}c`OMQtHGEUnky!Y6e&e44Z-qM%DJJ`zFe8No;|bCfYQgMe{=cL z3)bhY=_LgZ=j#Ug{VVOYtuvC5XoZ8lJ8;hIi$|m*u%r zsa(Ng1Nsotw5xlHd6w14MSQBG=NgS>G~u(gO|J`NIUTx$v<*3P+Dd!364%4rXq6W)t)+YJMf^uV6-XCm|5PeA&GC5f7`^iNr@#Slc%Q|F$aQWo{Gg{Q6#f2X43jdpM zmB1-tF2_ghyKGB}l|WEReV~fV{iq|-u&#D2aG6tnPO05sG>OVL zm6xPgbB}V%T6X>m?BefL#e{^<-j~0JcbO_w1WGN4 zKClM*Pd%&wR*DT^&jlHrp`_bzO1VvdQp?KmSu!&-3`Q)I21!PY4?Z0Q9I?}kKz#<) zBp@_~1Slf$YJ80Q{a9D}F@7R{v(8jdShFBo`7Fm-su1c#l&5?8&^;I=1 zS}RyZR)(Q${ep$Pg$YG$X5RkR*^7))b=jicxk^pLyyUc;($v_h0Q=pF`PVv2qsrFS zmTq1)q|B(#ad?-{tX$ynwX}(I7VO?r>so~6q_g}21F8JmR4}-BV^L zyAHG8_Ql3XsLsz9Bcj)`Bm0#vB-fP}^={7a_|yEJtc3PS2|K#Q)5fMIG;-D{nd~QW zB-1rDnT8(E+tX+DF0AZNwwq^^PC_Mg8nzPK6%SwWM%bzZ(kys+$sR$ia5P&9K!?}? zsp8CeTqCPCGeJmjWRA3%WHfmhCK(Sua+yr5ROXT8YBhDcTSmU>!MkMi}-sbf}fUjOT3U9$a2{%AS={~i2tEM7E;rK1T&756)u$+ z4cZH<(=pP5k5nHSFesPDR~hdgQT|PNHDLlZ@#&wW(}U}v%eD6r_0jd*@8o~yekcA; zey8s87~MX4%()52P@F}5&ixaPp|}Mf6EQ6~CM-6jeiz|1_!!|G$uahQ&;+4@30y`j1Y3Q_(pL$67_78?C1>=?buU%$2P{RNJ9D^&0Sue<<%3=F`1s z`ZPDiFc6y16EzeE#_I>bCL-%~C{dym7T9&^J}Y(r^+EN=jrFr<`aCwD*X#Fr%se*1 zxeR=M`X8Qv<9Z7xfS^ih^J!9Wq~VZL!R2Sz_l2rMO&7*#;LJGPhXsB2?&#W6V~hyw z>DqB$Z{fBVcXn>ft1OGJFh@9TdA)U0HqM>9$bHehaPHg^-i zeybSY^1{O#7H-PT-L$ad!oT%>mBrmrGCwD`J|T9u*}UQB5AXf{_HBCX5hvi8x4YalO&{|GMl=K!P69_HXJ}C zO4_xp#nPgKQnn-6NggyfKiIIxWi7LY{Ai}-Tm5KN!%dK@pKVgUvWxv_B`SS>@pnUv zxt$%}p*+I+wkW=}W{d8x$}c68rN70-o@KAfu;K<5ZBYK5u|kc&L+MmvY%MJj_%Ul` z%H614GPZ=xBa`Ogc2ajVqmJg06o18efZH?Mk>m)?BdDFN|5LO0!SE!Ct~3Yo9ceG9 zmqYXt(l3bh9U2q2Q^(Mm=VO4)`j3%kp6r&kr

oP#>71t_4~XJ0(%Cb2CY=t_hXh z!-`2deIEjO*kDQQ(l$6N3!R~JvyjTd<&Q5A)wA)9zM2K13aP8z>etc9jCMP;PA^}1 zuSHuI+%X|Rc|zU|xhf)akW}#l^~hDZexr9sSCVezr|{Y&Uy# zrNe0W{SEhNNvZk8`SEh*j+0sU2A)xkQ7Aay~B;xUXA zSQ|NF5?a|}(h?cULZL9pWywZP3sqr+Q7J>#F?1l|AlgHR*BN$p2{G!l$eFj^z98&e zDmz05*liwleklny@>sUmWcD5H7)#dhQ!FX%N`%fv?c88H5|q>Ow=q96Zab-hC*TGO z+7gARAqs67n!3TUJ*Ek!%j$6Q4p2;siP(`Fgh-O6ftwp_xW~i|EL(Opv%?3JgF_!5 zV5t(@q5N9;g>p_x=v8*?V3*jh*xw9#t3(SvbiKIQ+VaU=_GaW??I%rEu5IZtdgbaoR&@O@oSaJqO%E&=^0bwYMpcP#Sd}Gl39#t z=r>}#F&3PMEfIsyEgjJJN}I|0+gV|IH+>MiI$J|8}2C5=H|ZOOR$Z)ezZ&f!in z8o#Q37x`D}cilhE81=hKFou5@deNO4(~C}h4&CMXbA+D?zZ?I%c;1SSL0)F;`|$7L zWB7Mr%x}kk4%hwabEXBK6Xx%!?@abVea^JtbHc``?@abVedl-BI}>~t#WdCL>I;5X zF!w{Y2r&xCeP~P*#*D~aXm*CqMw;NEdo<1lj)soKyP{Fj1rLjx7Xju4C&fS~rAs?0 zZnq$~J#OR)@Y%EqoES1MS3EHUjU{SAHu1RuXms#1+4Xs+hrpv!TxX|xhT?%B*(N^- zOwQ6kBpR9{n4w2rE9m?JEJLHGYeM+){&PcO(QgHvDuX`uVYWGa6={hqpEXzYjayw@}2q93zX*pKcjL|(+It= zt#w23oc!EC^o}XBlq~25B1z@*`s{J8xAq{ju`^)HNJEGXea~@I^?4JaA~+(JAT=Ju z6DVXefS@G0#^Ns6fjLq`UO~{WLe;)LpYSm5;4u6gKNR%cBgfOEWAMd5UpV7EUcbeN z`lR5jdEH(Z%oWvqVRluU{05r*%Z5~2dRFxOl#{DuS63TA?kq7z_UFE^VoGC{YUY7q^=N<_H$6;{J29WU?fRss#z ztym7neZrM4*eZXQ}|1O+Wo$QeomIH$M*0H|1L z)`y4*xGmAWbl5)uU#ZPvO|)99Aj3xMa&9&OV;r@r2+=bQKn_JfO5j13s=P7QfL>;K z%0odTI=P#@H^z!8$=%9#RWk}d66a(u*I{!4Q7Q4j%c0nw#~f758PHI$MeR5lge?ZY z4xRis{E3Gng17=0j=VZV*()kWwh#~9f`W0lgl@LYYTM5KveoKfUF(&)6%L(V z`BeF((Gu0CT-y>WCWD(Jl>Jk_HAK!aIf?9EWmTkIG}9p$L+?&#yM}-zJ=ui*Rzj~2fF_lhunbk+91^V2 z9w1#tL0R0(E5d)`blSximiclI94@Inw9wbhzPr>ve{N>~29IwU`(C%--wl)EVmG?V zeR&%@SX(^jyA|V=7wf8$vMg*3p`M_c;sN#Y4DNEbt9CeOtFeDm z;n%D4tso}YG~C4yS=1L5!tIqBBAHn*A47yu1b`c@xgdwA6zv_AbuTC=s3@q&&&~1& zXCF&_e<|_{i*uFpXBM5ES~kchlu1`y2xSu6;#%51{hwJ7rG@QFjLlp)&`LVC9CJQ} zJ>c0f$JYT;AAVmaT{69e{Q12I6XipUi8%DL8og_*Pr>`7_SpOmk@DSe%UMtP*J zDy1NPcWRYC7t*yEB1C6<_$(9)w^8Me&a9q>`b%cm?LxtfETLI95Y0gJ#UiQC4vW;W zwKybE;~kR3K+&j=z~+C>XW9Zu1%MY66c-d174kEdi!17h{XZpRcCQ6ltSg_h zp?mtrlPsMhbkv|IwWH<)%hLt%CqtPyZ?KKQvs5h91!ia9B%$twTu|-!@UznfS;MJf z=B`nP2u~;$Fvr~N;Plez5CyNxSAC8wa_+6SSF0ze`p8mMmDDZ_S0yS|bgVj=Xpa1f zjQIgDCH)7^jYT*Whya)a=RnObM?xCXFaY;H^e7}+CIMPRC&6Qd9PuPQP*`eJlY4j^ zhVCD4U?5!`Bf}#^RCnA#T4vcISmM(0FS$r1WlV#V+tvC~OQU$7lI4gW9UAoj=BFT#(X&87`hwr z=OA{b;o`#XJ&no-w9dgXs9Ah__76J7&O^ zZ4vaqnak833wm(;y~_#vgh7@NwZ6hf0=epF1h-Lq{&~Z(pAH*}=WI?J(pChHK&4t- zy4nh1#J<9EmH-UwwvX9eW1N2dP_%e3Y>)HT=PBP3v%0!SpEWAia{09i=`5P6^GJe? z5@;8|dao8p7pjsMlwu>QrHxwz9AuqA()FQz7&2~p5Os`}>Z0M#ae=+64-)pESpDFz zs9(h=(ejg@wNW3^4P5C(sXV}C#|o#R6_+jDo*r6b^VqY)*I42?f(j?Hq?AG!ce#JR zaDA@lbWz2M(QB?;JJ(;39lJHN=#bI3G8kDQCIpRat9)tghCBn?kHr*!J|0SLzb@8)_Iki}I(ny@br z6Ng$?E9j4;M1mS&b7{}KaGStIL8OHsSD=D?u(?w;t3Y`?mDG*E13q;esxz)Qkk^1F z#G_s$I+AKY)6^&;Bkn@ni8I2s5G1t-tyiejjvJ?)I5A_%xfu&+FP+BZbyD|GYs#uk zr89TMpK|6eRL)LWURbj#w4qD&H7)s7OIPa)bMyJWNM$kDg}3)Dq(J*HWugu z$eS#tBYYvlKLT-GA()tCT^;W9xFE>TeF$R6AZi4KrC!&Azy|ApTzjN>{(Mh{&Fe7` zdL!uep)UyG52z#%m;?=@z!>7=1}g_6(@(+Lj!=%8wbYBRORYOgue+!2m4}NOTc)me z8y0((ZT>=MMeESt*qLuBJ6?YC(C+pb`P(*lUDon%Z`p9}nwQ0$^TaIw>N}U*@|4+S z``~0s%9{TD`=f_`#r|_jjpACSoNcYxG^x}ZWi!t*B~&eFp1XG~$#pJjdVSdIJlOwe z50mxa?K7xNYt^Ev0vua|tV1IN;v0c&C$cyce^O>29Md@RB4;XBnTS!D10Rxq!hlW) zSN9RnQZvEl1-dsaDbeMOi9+=jnv4;+fkXh2WQvZ^zVmhx@U;!7yX1@c$s(YNkf#M9$1=DXBlwJ!#gu9mNZ3*f)nhi*6ps?lo)wG+jU4ul{P=!-|_I_dh#Q*^+B%wc*w8#71SY#r(un&u*L}VZjai z1>Cs}`u&0)YpmyMth!z58V6bXuJR*%jj787H4vp(eT0eNpz0b^x*0TfFu+uzgUTjm zAXyFJLc?YA@YYw)1LpEbyV5N6qa#3k?2y8qtuc6f#XHu|>bkG1v?X(vIX7zR)SefV z{LUHw{h2!CHHMewY_9CEh#A7g61V$$A0!3=_1Zz zj<7H=Ki*}O2o0nz4c8!7UYia=5XiuU`Ml|Ai3v^z_^(Znq#Tx`!DL91D!dG*aX72) zH|PvFekyXP0wk&Um0m2r_UDIxbWXX%9_)H-{rtQrf3&MI<*#gIQ^&pC_g$0ToV2-U zM&|?ZwWl^8y325xt@Sj`$eF7b?~INU-}zy8X7lXLC!!;sm{L~IT!G_(9@)~nm{%e+ zD%cXxlt#vN6cGrGv=Fy)iWUw4DF9>`{Ip)}ToGY86}ts8!D=KQ9d8`GmNraW=->x& z0pDgcGv_?Q0LGb7?u3Sm!#JUm;!p`QEMtiUn0QPWRm}L$qo<#mlARPCh@F{QvUW$>8Hs@k( z&J%wkGZ0Ab z$s2ckqiy|D{mN6y-#4eVuG=$xMr&@@tOG48?mxL+T;A_pcdVgyd2Vjemsmtu%IpKp z7yg5vRct!+Am0B6ftXmly~Tj61*o36k3g44!V+WaQOSwW3gD5dTfmb6x>a!GG;A{I zKj4^yLlL5aRheKcWfsWwz+;60JogHNao-#@j^R5QZm+?Ii{M;PQ!2Qaz-dgk5R@4w z!ii#X2K%B`@*|7gxFJnS5?#u9HciQj*OQx}Jk5I8wC_3CFW58u?x|EOF^kc6?2_^i z@m0B3hy%bOJK#s%C9(nysIZ08P&d4r5~V{BWAx-(T$Fc~%2aYb@em93b5 z8Z9u{=V|d#mw}}4U6u5bn%H`{EX+m9cbC;u%uxG-2yEvRp>rgm+q z?p)vDVdwL{U-{D;6K%SWQfikR7`XpEeiu*uN&E=9BwghVWpTSXgP*j1x_L`!KUH&? zDj{B#4!lD2i4(Ge;j!QxQ3lYg;#F&ev;6p*1)+7`^Sp=F7Q6~793~|j<)FxYqFPI--{>O_};De$e(|Y^zHUj zpR$#Zkq%vlEl-l60n%dTBVK3}2R-;wyyjzJ1@a?`D0oSLB1W>99$fi0sxePyXM3!Y zE;*lx9+CT{wwR%?YRCpviU`wE8kX9Bq;|B8bB)jg%Ko3#`>C?CS!sP zlOW}OHvWJS!u}gNFXlxsvlCL6z!A{`B8TsA|fq3 z3KZ*5k%sgLKtO-+H0{D?6Bgk!`Td6<7VbZAKz;{*s1S%B##-@vNM7_z0cEuoPF{&3 zAe=ZwZv0=c;K;v#a}P_#qKTX{y6BjN$1FZ90HzueX0SitciC0$Pztb2xQ7WKxPl3S zN`T~{&b=Oid8OWgN36_ zS@;KQei1N?c!FHnqTVCXHMXjzsrH0+mViXO(rgk1E$kfd7@$N!kSb`j!cGiC0eF{U zFM~#H*r_t;H3t1om0r^5i7ZisYqb*}s?n)5m<5OvDfZ)w$Oh5TB(g3Wni^E9w${e_ z3b1YF=A?rPAz_+rnq*a5@%u$=vZhZh3VB8o2b+0 z+cEtFkOvgh)Os?iMA8T!POBCuhD9XH1&B^v2oOG)UPVJcNPv_$8AR;_@>8jT;Jo(x zh^N#Vk1`5y-;kOyA~h_X$Tg-)%M*=KF}H{a))B@;5Ebjx3uT3X}Ii`N3JUvQzXPb`|T%_Qfbf$sb+E0?5u)?H@!I0`{3c8 zu1evj6T1@g-@NDBA6Jn~bizl%N9x1)TMGOr%s`Ga3R=;Vg#(2~Dh*@;mzbD2@x#FW z4CC5UL=Tn?v^+}v_#!cMIpY-qV4pX`O^wZHPQcG%?fUTAL6t#=LL2ZnkOFa7DfvYf z+ovGI6(3(Gyyvdmu%Ko|@f%foyMJ!-w2Tc`&$w{o^0an)a!2mMm3Y+qCVJR#`+>fa zjHY6VS#=Yi6x&mlT-&hjk}MnB6PLUTWpb5W_(*(`uPF|^@m=C$N?p)dTOjBS&CH}n zw%nANl8izs5@V?UkgK0`+INk>{wCu>sp$tOUwHPn6E*2SeLL}no&D_zU-x!m6gd0a zsfqHR{PwfGlUUp7KSee*D!l>e6w_{^pwSvYqXIZVJTqwwreX8|podmOpn{ z#j%Ik$FUE~uknmeP~P548Af2M=>tJiCmPQFF(l>x>_KQJ{tq66_Uk`;5ZcrKg9o9V z|IZ$TPRD=zApglTD5nfjYQgmB@fNZ$)4Qg3bsg+5KQ-YBI+~_!Ab%^?naR=Aou#3kg zmR6JwT~XB1Upv-Xy{5BkeVVtsu6=PD+aRvo+Swa#XI?u&imPoiQhn9QcVrosT?=a73U{d75w}V*HP)A91ZOq3 zE@C@dHuYB_)K?YB>onJ^0gyz*+K+m~`IXG9o82jx%xz3-f{HckOgb~}o=K}Sk-ahK zv@qY2L2Z!KxREMqZA7LX@G!GRM;8gq!8HDP(gMYWEl^)ich~f3Elo95fr`@N+#Gi* z^l^eg$K`5?YGWY>_)>|u@e-U8Wu$}Pg7#!(=O7SHsLTcrR{}-qWQeSag`)JLVxg22 zHjA}WMVc}9is$7&KlqURvyU&_{Shns*6vpi&yhBI*5AMVCix#Pt~vK7Onc>XEa?VE z@tm6c;@S&x*Ofi=yN$(5VZ?52*KBOBziemM#C?<2Y@Oy;UuKE--NMq|+qL62@~h9j zAiwmhEwff)Sh{;!`TAFw>887wle4u!_05pQbnL{|(77R~y4LDSQBS^ma*qn`R6t3ljVyqmk-WmoilX_ z;!0MxN;K+bvC+|MyT`A*@C3xO?TZ8Av(K|Grn>y{Q!4q9=bw|mb?WloTUR&q7j1o$ zId8k2x!&Bm^-cK?w=aHZolQzT`Eo)>hYq}I%pfx72E3JuEoNH!4tL6R4*j!3`UPgP|q| zE8p+QOi8kWk}d39k%QraOX}s~nLMnJ>lA+q4pjmrb;{HY@(#9g$m(goICM?#O@F*` z_Uwa?4PEqlU(I6!eRr-J`tJUgD;m}BoycxnWfamkPN5A;$ z_${@IFZ=YG`&iPw^RPw_*0_|fF$*{uuzzcSNW<1*S;&uT$$vt94gQlxtJcu55>c@y zFPxM5VWr%jk{3>1lt^AwNl|W2hMVXKPg^7)S>!z>3U+&G#DhxUi5loYYcx?RkOP_H zQ8$dY%Qq-1y|4GCkFM;Peels0`(N*`eUw(Z;IaJ$tA_Ye)iB{*iy03qD;}>Dm(?!5^s{U3m4A1yuO9n)>XdAerX$-Xs&X(NZ(C?f911_U5kgTm zeCcEiVU%V$z<4(kF^2%=RFOkEn?#%o~6~Jc9hfP80cb+2Mvf6abyG_6vf9Z zRm8)j8SX_-9$0X*r(6EwM&{f0o%Z_5c>_!nsCeU*C)1{-k7iwcT()Tc_Y0F0 zxAX12wN{~iPV0$>%8TTOn=6lqPJrs>f5Pr=|5oj<9~Xo#jLsMF^pML7r#_U<<8q0| zB-A6JOADAmU+hO|ppI$GX(%Ls3_!#00%XnpwIDC1%rZe^gn3_#DkT&VU;GSRAbc8YjvI9ksa^WWPbVnPDMsP`1Rw1hyQj1 z7wC3&V1c(@&uZti&SmVP0dGAr5%>@M+K}O$>K%CMY*i^xkefp5tvFo`Ol<}Oqs9%5 zjov&#XVwW2VEE90&IEysgt1L#^ml8_YUpPT6uHKLL4-I@z5F~uTKqhTw`|Q*>{G02VGE$+E1nywZvE8D8L3Cdx2S z%+NDx={EO|i;xs1>~7(D>*0e*!@ zeg&}C{TSNL1|cgA-8j@9lN>fAh!vUWAQOuFeTwJBV^f!dJ7bCi7A`&7ggNf~bBflj zTiKtSX3lV>m7do-|H=h-UDCE5Pj>G_Z-0AI^5KXM!!c9S66e=%B zP2nbCmI#<`D19spQtq}ZCp$X_!Pwc3cay|XRMFzhq!rU zQd_vkuNKGL&3mVss|TmSgLmA}6z1QC!7PSrpLH`3o=OIC)KyVC$kNl%3!N)sRm zpVbIJoI#CX41-z)4G^55qt{3Pa}b?1X$~afxN$+uHHuX;URY&7gQJ4OO-ojW%V9AC zpF(PZHkMn@Aiw=#$C^dA))(|?Nef_qU0PdIGP&Z|xV&q_4I>Y>Vfp9m%wt)6?Irm= zGZwVYg#Zp-^fw=u$xG^`CvJ8BYEHYgv+?sY?$Jk-(YOTqWT^E{m>B#z9 zlXAPVvSwtn!FErTwzww^dLeSRL`+ad)+-J6#*G;z1^I%OS`qpDRwTYOl=#B%JMvPL zj29%s4-yPBHUT($exj)RpMk@Wjugdn6vdIouz{39s!&((qZJW}H#Xx%r;I1SNJa!)3^)I~T? zSN(o}W6rtxg=?SLxaHZgq{q$n+B$g{**`_cn5EU+KGRgj9ieT&v50ZZpe`v!VknIX z`fQJ&)4)I*)Oxi+4_bCXt&hnAlAR_j3q&>Pu)t;^OdCbsgz5S*$Aza;b)-gNW6BEh zkVZ{P0KE@@97?v`6jrAqJbHMnqye7n>0Y+>QbCFkualfOrC)J;!bS2!%2wSWJ@z}CO15%bi`=CsNOHMVGs zwC2Jct8|Agg@qOK$8MHF6Vnol-$Sf~S`z7^SEN6MeVI0945j*+(NK^B!n|sxjjLfI z@Hfyyn2ZWNH1iQX?qnbQyEub>qbg3Ti^CxjEdm;bc3YvTPy&VA)E0O-3Fz(utb(qr z!Kg9}tBfGo*Bf=nNNZIVQEP$U50FeR`Ta}c$?u<&66&Xq*6HJR|DS($f2b(IN~gG? zE(i~*q!=EQ&+DNP-!!z)+Lq8})JyST`XPmBizv)UhSr6hL2e?*!?S>utI@MMQHo2& z1L<7pg^5mOi;{EK+=KmN&3*a*%$166c2HCE7nY0B5a1)T%2bW2OF|cC z0|4a9M8q1Tu2CZx`2~0;A{<}QxEqZm8l$k&4Aw7=V1O5IP-_h8orsVkryi~fph6(2 zM_-R7_OOGr3Wq&Vt_ZQVy1X&anC~Y{Ax1$ttYryhG2$eTo}2D7PhyN$f4X*=;{WGr zHlMsHDolO`W8C^F3PyN2;`@Kq>5F*v6eh(em~BzNA7>9adEti|n(b$U6#{!IO$N>4 zO#G=vH2`jeC5xJ35o}3tS4TlNgLBnStTPOqO%&}o>~vd;3+)w-3Pd5MQ*sF2cg@@$_Lvyl+RwY*_RCwmJAABU(*^zK@ z`q9DT>x-{XyI069HnZBafh*=efU3e<;{`o4w=|7i8Lyjtq-pz{s_)&>dD-T35@wZk zEhx;2E1h4uh?Uekt+Nje9DFovb1dp0uQ?T9-K#>wFu3YGG&Eqf=sf@&m2MfjhMRRBEym5A1rs1hSU--#w_kXy~mm&Z86RCaQb%$0ILSJy-_66?S z>VSdx+WDD{P$Qk4#7XtqlB0;?gK^&pubZ}RO5zkN{ zBgG^?m+h*%XJ@rpP45+eII< zjocHL7I_wjBUij>Vt0Dd8}iT2+i>Z~%RP|IoQ=6JvXw$h^U4`Rohu-qdF?wMgK)Yl zE?U|X{}Q>q?Dg79+b5ddXr2+cplEJs9_xdA#vZPE)c=qdLfZes#&?8I_gz=^$7>4f zAS2Sx7WGMfR*dj{*MwHV9RY9_?g;wX49EnlC4t7?sVLk*fD_gs2zEu%H@GiQGjw0@ z1u7Z(1S;8VqN7p;<5pOZ3qIQ~RyB;3*tqGTM z>v1slT{rPT)PWP8FeRIDa=5f+?Yse}$++!%Ykv0V#A9LqjdaL0JT`EvysCMjxv43T z2jm0cA)tY`kD=zgQwTnp0H2KJJ0S$Z7dWi2P;gk&(@7lZ-gMY_N{wUGuUqZrVu)BM zxU3N;mDcIXRMN0Mghe9vSzUHvuY7#&(S>sloHJ|BY#=?JN}JhMeW0rRf?3x7^6EYl z9u$>I_wN9KOH^c%k4t(1+ zGp0?zs31N;l!BZzGzFje5YZHf%+dLYy8(|BHiT*g5=MmZ<>Ctz1#0?G6tvh-12SlN zUQarT0bEVAs$2iLnouIWN@sZ>YF!*9wBM7OLxEi>>LXWrHmWu{o*7!dzHTreR!ghT z+c2xCLI?zN+kSHU;@g%jSz5oo{jIlZ_cflOKYrJ?V$I_AwnRbb8(A~3xKcQpKfPqx znzr+2hmK@v152h|d4u08|Bjyn0ceMhga&>N$}q)iF?<#^jUs^;ucL%3LSm4X?WR)! zhnu8BIRlawX^<&V%pi|qxReU>eE{#%SeW=Yny5PUNMvB;2-cMKD_(hNxXy$$f}^a` z#ZlKtywWkz3PJg2%(k?yvHsw~(S3zU-Yg+AuWZJgb}>gNpErGa)3)OBo#8|BL@1=S z_xRcenrc%ElY3cab!Af{D26o%+( zdbZShlE~0N{XpCsDsEIfe6pE=Xkt+YrIZ_c`1}6ntBY7Vph`2 z;oJycrUTB0aA@L?u*JQ!p{f4z<@=7!*w@GGd-|$NmNz#qFUgzU+3v5HNBb}PLrogK z`}gmhch#aNvE`e4d)Jg#S2i~UvPegqdPN-K`(CeF9U3h}dqxUF-h%ui7zXfdl3_rO zIHC}c^@L?WGF^vcIurvmpA1qPvBUC)nw#8zUng8>gv zu~UGtrGXOK%ZQhSu9k8odgoSwk{ottu2?7E+;wzlSrW5pY)c!q?V2`RT9}_|UbS^= zcf&dH0GA{!8@a1%@c8nqWrN$U`9)Bge)1`IV(RZV>FuJyoMg=^VPMI!xjFF~ zt8wf5MXB}cW)&`;o2*1S`N)P*zGYDQZf@{e^Tj zWve;8zLHLe#H5Ve8Z^JqUS$SsID`&YIbP!8d2IAIxWM65wpQyX*D`A+4$;1e><0Oo zLGjR9FLRkEPCUQKoah)Cak%0tKecVQ*DYN<{`e}Vt7Tql13Sd-mEU+?^X`{9!e2(m zUpmHmer!nljI|^i53^S;Ghzz%3Rb$J`j$Q&OS7DmH)#4sP-K*4VBM(}2=|m~uOZdB zp$+9Y$H*gbFIZ2C38^OyCUGH4K&Ulf*VKB1O%S=Bmqa8z9}`W9z@%>|eO0I)E8fanG`X=Aun>;R}HIn5*n(l1M z3=_hO;Nki*+|(I{x57X*}gq5BiiClV6z{m~0AztqKi zWx1kYFZP#BUw7{G{VTj36PdO|LDThVv@8JGoVw@Khhf>YtKJADmbjsjFfd(L;Y8-Y z2{SF5AtW_)S2D6}&1A=cP1uc{H>=S!fZG5j5V$p3Fv-KyK@m(6&HW}OK}StJ6N5B; zos!@!Q$K`P(*E5~=?`VARPAkz^{Ax<%ggfpq?;Tl(#9F1M=e@+RLb)K9y5gVV>owb z*@h{VN5a)=`@}0JRP1wXgf>!b^tr!sPo|U~u|d27>F%OU`68Sj8$8S2$p46B&E!XE zm-xAL$aje6iT}V!$|7IWOg+F?*u!+Su5H~1rmWOuRx}Q8=itw zphV%bg|B5K{18#bc91*CS5D4^6PJ;n+)St@hq!#>yzzGiZSnE8!NLu@D@J>4@vK{) z2*OwOyb}*dMwZ{Du^T6>1{+RV!zEkT5shGjFBoUUJxr1tq&!|<%~UN5Eur|NnT z?%(CX16Ujy&;o~m%7#u4_$onqHRw9jkU>XxqMl$>C=E$e8bXW+OAp5wE8u_w$|Gp0 zN>*7>0^m+&dNLpfG@pm7k?8is0HTx_IPBW`N}~R6gCNFc?CrtIDN7D4%ft7@GW$%iQkI!P&!lDv;>f)>CwOzcRB%^V~wmAK; zCNCw}RyQyxSmiyIlnaN}AIQWJ#lulv_f_sAHmGK*ejjosGqa`<74r~iZ~RP$5fcU- z=@05Xn*-DVmB^7zKtrKvr^+N~OxSOOPSjC|iKA_#>^x}Q%%pVz{{_J~xs*uKBj$wr zXza)AYc_~_x{;Jg*Vlg{zbyaqNA`!ZRH_*rt<*q}0g`=(^%r8%}f2^9+=-W24MzJ6ki#B@itBIBe4DCe=Hw|5Xo48>T zlqQ_|Soka4SIUb9p!*Z@naP-igNqrPk{zaA3fERzP75mC z(Kyd&g1@RVf#OJSG7Q82Bv$}m;2)umg4{<+s4ezm3_1>K|Mx$qKZKe+@P#x~vH$}; zQ7=hPqiTsQA znXI5DIc=_|Gw7~lxpiqHomulb6RUa+*H_**v~_E02y%OIyw+VJp=NQV#oiW>Uq}+9 z68^ep$v}!HZezns#s0Mo6R+amxh`k2S)AZLNQr7oXcH*X(3qt)*irLHXF7nEC}svi znJ|DD+CgPd0lZ@^@rm=ggW8ntn!$|z}!(oO>~HbBe^JCRTW zUXR1rC_k%!AbWW$1d@)t6@q}h$nK}5BqLz4hNtY3_Ze}=81;yT?Lhb_oK-GBgGu}4 zD;m~7b^5BKl-f~@Gt?R1AZKw}0FW(!7)a}_Ya6~aKJm*}4qwmHNrB6ZV{ zTI|Ia=tYQt>%~fXpT;2U{`cMp?ME$IQofXrehQlUiJTPqwB*8bQxrjw`VgF3D}q5N zL0rDD#fkZc2%jIkk{{hSlp2zMUY)|PR9aAeyVgC@>A{u4`tKYca2NT77ZOFWq(l@G z<<)VH_6k<-lIXSMlJCWR$~MdQ;kQqo9dv)Gf2>q~?Zr@A1-MFSNeQBe?s%jtscnL! z=ZbtUDM2u60fIBrw|Q3uwOz!1TSMPQ!aMwJ>Z*?Y_G#}XtbgpcPkVR#Y2QBW-2@`} zC-1hM_U+T&?Kth*r~Pgk4H5g{r@cGzv~QpG?xfScJ>}g~J~#G_5uCl!(?Rw4W)+T% zQGqn$>=5?U2<$1^et{w~BA(VktqEv?Azh{zk7)Er#+@PsWe+>f_#}`rU|hXfG@8Yo zdZ>bb{qtzk&-^?MVg@`J0BNYA0y^-oe;%EZGe7T}KOY^dGe7T}KOddLGe7T}KOY_3 zGe7T}KOdd+Ge7T}KObFShGD*F0 zCl)%Kn)|`o)#y|KO;Z^e_C$e5};dEBzF3U}fPGOxwujtV zi&=u`PJNK0456QZ9w!}o+ zI`+koEwOHM#nyGmUU{aRiKKX;=G=Y3oN6~)n61_IAp`|HG{VVL?BWHyiBc*%L4aUR4aFaOo9E_?As45!5 z{=L%b3_q9q_XT-5*{Db0G&t-r&Cbyn2XQ%4?WmW=wuV!XM*$v@$3{wpx#H#HhuQYY zCg;(Fq`4<9%O7IrbyQT$oLNzU0-*NWUrd|Y+RPii#SODNCcFrFq+bR)Is*7<+yTJ9 zh0ob#nyc|Q3p!M8!hAri*^_}0t`TC#GIR{64F--vq$rN+ej4&6!T0(rN#D4$6yFe2RIDIA!d`5EAZbQm1&Ju?QpR%kkMegv9OgC9{%!3pJe;BsN(aW4>fplFDmU%;h#QvhMd zHIx-2$~CH}gtcO{DZr!QhC$g|O7Pm*Puw-hK4(tw*rIvG`Ky&W_uUwTl^tN;FV2-O zUfR$?x(c;TLCITXPe-ZA1=vMtd({G5#-FCc=)7BjM4b_0TA(7k~_3bF;rtKP4(gld2v^p11L*15ZXLm6W_5c@cb=LSsNJMBrPC9!shnMfIJ->XC$|iC-Si zUe12J%yaCRXPF#Wo_*YnNtu6OAItZrDyfN7Hh(dzOBFGd5s6&+1?9Q_Ec})2;Id4o zvw-4{K*(?lj9cxh)VNT!z(l?okzWEn*Mn7p9Dr53RelmO%H3QqRs=enx86E${OC(B`Kw<}uU^!+^zlYdmJ!LLXTNCEW`xz} z`0Fd#i#Xq}t&~k$p3FFCsH&(5E?OGA{F-`|bZRGimwlV-)p)G4hH_|`a2>L-S`5U~ zCfWgqMI{gycoyWs>q(eEhZ2Gg`cy;Bo&c?(QIPHi%6J^AhA)72Dkra`Sb@G)RutEi z)Z_t=laX#MN+>c-0WK8ML-reqGyRikl6bt+Xtw#LPtkAHJR%OBoL#1I+au+hd62`rtUgK zuvbXhKRg_~WAUCn$vJ!M-um3e&Hk(erL-VSuV?Yax!pbm049&dN_zj z!n6NL&X#WF`lEt!gHRm$B2*7(PzjRwC=TpK@my-oTTf}7Z-$0i;cdvRC1 zb!hyOn&`!0VVqdu>M&kPb&j#J+kPjMg9uUZjWM*1m}8TB;l z78zsYjgd6gi=1x9hzbfBQYg+?wKZ8eA!TKx9nQ%ne;T^|>J~hAHQyggr zFL(e+*e4Lvbq3Y!Wzp z1Tgc%uHg{lMr+hmia@qb$pTaSpb`nufvZhJ+kisF7hG+eW*8OaVwaav5Rsi_txl*m zqdPU+RS_}Mz;2@=%mEm@t76Kvj+KHve7!l~NE8PrcI6e7&dN!zNwq9mxa`1yyLx5D zjKa;Wwz!B?1Xo#d$Dw{-Hy6gnp8WioMa;PS`guKjLM4|}2WK9uXg$&K!HXiAjSxv6LSKjG4v@gYp8jDm(>;APLDf9crtP7J3!w z-smMF7C@Daj)@UDcb-z|x9uB!6f(;= zld2MczUbt+)2LueB?>91MnfHpd{n&>1U7O3fsS$Fhzh6O!C+-V_zv=BYrFcXi7PI6 z9@X3JCtvn2wkO3UWjUP9Y2PWTo?mg^_gczo*|kkdIPu zWJ4%|WrM0olb!y7JY@~ep&PI)Oc zUSv@A9hEI|+>Hh7OT`xDk=g8-4j3nd;ZUhG+8g(ZsU1INY}N);bfaHJNenlvtucen5ODw#36`%e%VL z@4doxgQL8#tat0nq`yz0wxh_$PrcKaA)oC?rcvj?NGfBwH-GkgQ{#ifP?5jkCb-EI{DNBb& z{+Uc7HJt-bvm;+83v8>BFRJEuG(h&t9gl*Q3povqhl$2gABYGf8c!Yu1*nY@%3J8S zfO6B5cZ@=lh8%V*A6h&XPep7;xQ+nj@g^~UE{a=_fYV`#i>mEtVFsBZn*Lonc|4}5 zub6yVInGI$Gh6MrXyht6z>aBJo8;M}Mzc%7-qos{$XmklB?saB2j{i;R zackJ2n(Ai@3Wk>K*i#fe`5L!bJ%r<4Y<6q9((k*#XP zTNFoIN46NVV($bpi=bqz+>-npbSuDFnO3;dR#r}y@ZGu!{TiIt?Xi86US4`WAyZ#|! zy`ACxL}n)rcReF5Ni@OHgp)0TBaryKkk}yO97${-Ai>#&od#5ZF;lO~WMY5;VfYUm zl5{sGo2K9oR2vliO*z{X|B*|gM9-w~D0~VB9!gz5C*T%t`pX;m51V_-oA@s+ZUg%q zo6V6}!t|BfhlP#k6U!XlzEbWpgVzAs4K#Jio(d+ur!G}vsLq1NM9hjEp~5^OYl&8) z6y3Rl!8tHM62pU5@TDN)<9bLF4ySzpYxB-RoIvPi%1!76(e1UZbEjwn_pe;E-`}GM zdVNNQc4_=M11aLsV*mIXJ8mtZxb8*Z1zNa#3xO?y*GEs3fEJTtgd{q(TINN z*$1u@fQgVeLdhJ;g(p&!Cq?{^5dtwR3)G0h6DpMz7*?FZ5p&9O)Aix{6I>-h#Bo~X z263DOB`{oA11*BmYFL;kqC-+aVb_W~<-JYkmXvzC8oT9x+>~ElFwk(${+adHUYs}3 zP@Q(4$Cn^IH)~^gX;NwUhW%D3j+<6epIdGbS7STYISc!mr;paCCw_bA=jVGHaw-F+ zWA=e{qvE087+Yr-G+H2g+9Slny}VwYLiq*oN2(F=@hu9s)hm$HVR(tCq5dbK``6jc2&tAHrw^aW;_h#LLSgp zSFpF@mGX7G+^hj*2<;zmm9Ha~$p^Nkn)nnlnO++x0Nn@L>p<*^3crQZ)Bt}BNab!M z#*zM3p@$O=gpvF1+*tGs6-j2n}UvQ zbClsO@yK;Do*~1?`DY^dbfQX$_?N_K6vIA-Yr@MvT>xzJ*d1eR*QC*x_uwDo$HPKU_WEw_3stEu z4($iViT959l40-5&4Suo0cjvtqty@*5$K2r7l=Jq(-EK|JRJelj3P&XGCITX3qh`f zuC&;D%`0iJA63E9VnD*cG7kR`2q?G*v3GnD4vgdlwJBH?1SLlVqd6$nNuqlaVISgk zCAakAI1@kEoIxj}+vqZHJCHi0nO2rgTrF!S{=U_b=uFynaq2S7jMDUSSE6IeVZo5| z8;AWfOj`CxjKJ>G;2%4Tx`{^=p~2f0K9|;W4xJX#7%7Pf;Hqk=4wU6)XJY0$pWKWT zj(Q{HKn>hzSRgDE;(c&=0p~ReE;)WR=xa=80%kFm7vy=esA7WhYoxXqsh31DzX}nA zlDq<2YPen!M$AP@NT?tw5W2~KVA4AI+sq$IO`Xx6-P@fK-%(w$tsr=8VOk?AtVwot zzihRv?RvH0!*)zS^{0OM0L@slM1R_e4qu>?%`IONIQs z$=4R=v?}=}yymvRh|a9!S5sM0UdGRu-|KOw zg1-TrhI9ya@+Nt?ukMnM0O@?2QJ|AuIdn#W`pTRutr1D@i%jjJmU?DQwE?GK!{ zsc7C8RId72Prheo&9d(N9`^m7{Niop!*lX`l-@P=*(5wP(jgFRKf1iMCAoaSUz56W zcFRQdBy!L0Thlx7hv@5LNf{fLO#B_Mk%L8kU-K51c|2Ow#=wwsclVqQQM3*rzDrm> zzfmu#9l}6WfiD#-&hGI(&fKn9Gum3}aK14m zvM3+>LYF@otpGH$BwXuC3Ts0~@(&M#z%)dZ%0bbF00^#4z|pcY){>}^$R=j z*B@guYdceWq`ZtX_OxnNa*w_-e@N|I^K3WYPq$QZ@^zsz&Tje@-=md;V_+RD*(!e+ z*{;vc38wdz7l1Ud)U1Jk%^Ux%sk<#do7ETaX2>7o2iYZwR+dS7{3!e@#(pQ+x&G-; zeJkX2+RSK6tSBk;q^D{$>c$3SoQVU8kaaa`uf0{m9-M$8S@QieiXZ<}~uZai>&gDLCi|X5<_-q?cz%BKHMT@E#oFqm1 zRk%pt>9b%8Bj@W=sQ_Q&DPKPS4GtvJS|k)aqK?ua_a2CXioaEBAvf#advAXzSEXud ztf|6P1Na*K{FKH^bvcwu0oa?AObj_63S{MLAfhnB6sbqu(@2DYf(n(8L}|l6DvvEr zZIR!sN_EZ62?bNDpU;%2O-l|AxVt+3xAD0d))WPoj>Xwqt65F-_5J?7S~sjuWqA_Lf0}I8e87E{TGhJH>NpdnX1rNWvl`U) zd|WyxLj}!l%G~kD90*6Ff#O`)Uzm_#+y-QCw8l{$-u6>?OQ{i=!Ak@C)vY)Q=~Mp9Gb*$rx;p z)N?XT`uj=My-%dXqkW)w!^n(HyKu274l~LwX=k`E7jj(t#&64r%BYJ`vZ3!N9L=9Ra;GnsBCyFkQFX^cmh=L2uCmaeqZ$W-;PDZ+tP9eg*@Xe&I2qhBY zwhtwo2&<$b%Sl^D{;%SuKy;wtRBT)kLgO&;fJJ)Xk`?HZ8s)ANUf|LxwfwbNd)sb*{ew_|9EgYoJ#mgfDQ_at%%9&=?Zt8x0` z9}de|em{Lhtt-zWze1MkRzqpQYsr}_mhv-rY7I1Q9Jf`)svAO9oI$l%?8OmNg9A1S zpD&@Qy>$E#Ut>zedJeQf9VFr~!v%~mz9Kz<#3IIV$G&AsE;8(7pYa~dkX5MwZshbP zbW+3OjucG8uEvy;8e2~!Cxf!q%n$&^5BJ}4TU)-3!N>@=@)3$T2XNIq> z4#r2ZG~?I&Q& zf+VCRI;_YV`C!1s$Q%Ox9r5(<5H>Lk>KT( zO1cnD{gK8U?Uc9k93qlBXZ~6e^Edjnv973; zzcF&0_$4nY$U$o+)t#bT(2>+3d7hJ6GuCX=<)-%27z6YqbMo+Lf;qWR`P73|LzQdXpB^eOm93A9%mf;i&(t-o^GURN!_( ztpc<=$c`mP0Cxnhw6H5>pI{^)n8f*)U*-EB4@96_)r`p-SQktHr)a{{#K53{qveR& zDD8V1%Bj>s9i?54R@0QY7Zm^q-IvL?Kn)Hqc%e4oDff&^mtXzh16N#e|0VKmn)&nR z_Q}WDa;dLp{(Jy0LAuP&={Y>_@qPOrdF<}~y?d`&c<2y4s}tY6Q*$|X-=Rtao7TVg ztRCRxsVy-nH;){$K_mnPBloh@Vq%r;fOn}{k(7yV{dicyt}h||4q|JUb%4ZE0?|aaQ*GydH1o<19*0e zaJTSN^^W;DShVtoH0Yf3`-Gn$LVn$O=glPaugK)7P~ulBTC6_?Xj%Y`ySE?QHKr#nlfM z?|PafHhI+ zMlB3&Tt>9tBlQTfALQDA7a`mY#`N&y8GS{%2IW0e#wshV@ooec z?^)-~T)&`WW1eSG)!Ze1c9d;7w{@#0>HI~52Rk|rup`>eP*t^p4DP*gc-b{quUd9} zb!TD0?9!B(i3Kw^*X!(>b(b|(=FgaSSr7it4s;C7uS>yw;+#%k_aIqWA*nxx3;`y_ z0SN*nG4qiu*rqNBF)j|FIl?=G5py;5dfoaM;@LP+6*p{RA{b~z!>|f{?t&R54ZYTi z07?(~=rb8;yQh34p?T1ct&e51{kK09)U(RSHE>EJcZOc-e5xu?_KbiP3H7#m5L`xv zuMy1;#axphDIQZ{bC3-7AhIeY+8-K?i~_9|oIl~eXDT$R93+=BQL(ziJ!ksZ%=9@O z4a>aj_tSKG(JIR~7(`=J6NGWYuZ@Crrrh8is=wogTa^77xMEe?`sO2JZ5vwZ-O2u& z^7E}98Jb^fH2uUd^H6Sb+7r9?KhF2XjJXhVd8}25*@cL;;&Fl-175VM6K{VWQ)|(i zp}|NkIB4jv#r{zLAx*jgY!KkVaMv}@W+4Q?EIZ`a^_ZvuQej+T4%EjK9F4;0jq}af z>RX%ZYRXCoQb_qID9||214ZyCJWS;ZG!Af#DG7C$x$s0G?inZ}BZxzQxVcGNm0H0@ zi>u%?gK~NGw^r4y>n-mZzYM|Z=i6+_b@RO&o@!37ya=oT3kLQLRlGLs=!VAOhQad1 zyPA}|ulVQdKfEBPJO=^m$z8dkdCnZO;Iy$)Od+e6?~8K4fHB~g8@8-^_S_8N`|0kn zN1&fieFz-pwOp3@;0>)OY1)`sU4#Z%FEd&)(gcMwM>5k~d!q$GD?u|2h+)ZlAbb;u zU^F0tQE5zC%rVr6!*PJ1BEA(su{_z`0BQhqH>gy#)tDfGCaAnz@C0&X56Lp7KpW}j zCjfw=E!nr_Mw*7qWCqPaNH}Njr3GdEM%HCFt;Y-~kBov-4~z zRbAfSOgt}_R}Yk5aDm*&-jG83S2r#KQuC7&3F7T5FMVyjB18Ful@rnwF+#zG#?#$T z{Bgy@V;OV?PhADMDQB=jHJeGHlmceawXitwqiEt>K#PI8+SGtll9_0$$!+gNY15+B zA@bL2EmWn5)9T}PT3MV19tylO$t0;wyTP$(GV@8$Ve(CIO%j9(nG=;6lbmAne4K3o zVQk;O{>dhhbMPpT^Ejrp8XNP9j*Hs4tnAk|eKfE3yFvJyz%*UZbAjeB9 zT}Xj4GHT&59|7g`^Clg#+2QTXi5A{)n|M||dJjnBtd_oe@?Cxb)CP5U@`on_m*LpQ zJ0|alE#oro_I-ML^rnn2VT%X6e*GHkG>b-Y;fj@1`&D#S#Z1o z9s$R(EdgSL&)K0m#c7BUqW2?GqA;${;ou2r$YBz(h2#s=gBF!b9|*$m!*=1j?nB#v z-;uvFU1#5R=+dor-E`JEP3PEp>4AoZy3jxBni|>frQIiY3-ivs$94r0PMV@34U5+u zO}J;vmb-0FgTj%*3Kp+_+IH``-TRZ+`$_w6l{1p|cXwZe6=1avcHhQ`Ebv{a`B*@r}wElBWjyiYU}9WL4DcvULi_UQKpasre{ zrrN#59QB(c!%%p266z93VJD$avNM&`XtZ>V(hFv}q8X%w^d-dQkOofvmymqG>Ci3B zz54UZ8e6PF<5}f0rXR&O@+}Q=o8X;AcDn=klh4V zK>US!OmnMHQ+0UwcLfYfg z#!|Z*Bm{||5{o9dnDIhXE<*jav4I3Kt*xQ6u@jvcg*n;oR9ixfXA(g@PF*?-fDfVw zo7fm7Geu&&vX3`TQ8BriRVSaGlut7I6jiz@8fJv_Fux2tbGQ0-?7at!)!>!_!cO#NFHnMn=iX1%S$4MRbGLO~Mck+;SK!BL-j$pyy zM^4%9DKJunrICBzTCp@%3J*L0IS8r)`LDdf*UxJiVs3Q#a1!P3aS8DHM)S(+qU*jr zy6#%l(a_;s5YHRDz!h~os4H`{(h&)hLMcNv-Tpg#@#9;)9 zq>keyA%m7<$Fwwa47`ixFbQX@D;S{|fNKr*1?r*C$^Zlibp6O*q8t~#0+YZfN=C)f zBr3I@D8sIDY#cE@8rPsp#Df!h5{j1lT^PpZPvg?D#hjlwUDzk{WfCzWVHtQ0NZ~~^ zJP$d!qGJ3^ps8#|&V;DLnfW`gu0A^q!BfPkg2B2n&$H)coGzjW!^@?qc5KaOxjR-T>0_8 zgki4%f=ddE2|Qjgf*WA(vQN(_#T{w5{-A$BLsj~{o?M%n?HPB8e^CDx5b~GnS~6Te zHaKQ2&v%YmouJX)c? zVjjvAOkAC~bt0Ypa>$$!Gdji{%sF0A(7ATg{fnlsVYLGmwxjlqVRDvZAL5u_o} ziK=U*8a0M+QxeGt(=394tRl6KXj?JlP^r#OQ5(>lVI=r{z`I;Ugd+MP|=+0DBr)-3+P-A~@n8^qihY+hi& z+RdxyIqkC#j=gdRC@3+A@5Go~NFFJ%Eu>o5n{AseKDen(_>)j5sIS_4{CqraCVW#x zURA0V6=qx%11Fe+?pP!6Swz_ly8-%c#FGSHASn@PD218a3yWec2^9woOD;Pm>;Oqf zbQCyAwbdjo3{P&UZ7C|CEUe8URjI4Eq#{;EA!(;zDjnOk!mB~OYzw= z)kPCUjRRHgm#dR%GS*zs?De+pT30Z8iugYNc;ffy@ycF!XF}ra#Xq?72?fgFuCA%5 zG#t!$YO`>O)V}H9p-t^yDZ-QchhMsFv2>Fllpfi8oGuP*vmG3sTAt6VLEXe)LpC!2 z_cTJ*&;V6JAWPD~P|}1Wgw^4CnfmE)t}qgW(z{6A#qxQEy^a)w%4dFnno}t%)6Tx` z6T*Yvh*;W}ZveTg%1fkz?&v7$%}W4R-$8UtuWI)X~8rX3r=)fdn<9N-3)gul8Q z7A|V2tjws2Gql?m)^6I-xM)#hL$$BY9^d*v?d%H8n8$A~E!lcN>D*<12!!%-%KX_y z35K4dI{RREMN^)yBHvr%G>ypLzpNzHpDOlb=3b|b8|HFFoP;}=fG2HGtq-lCVTz@g z`35t`=Oo}b!eiZy00HhRjptJ0B%1vOU5$~8)M!W%1Bk8v;UU+%=q#vY&Phk8>=vdNRSv|Y9t~1qk^z*pPV+W=$YpJa1tgT<> zu|Fbz5|>kwE5Ftpyqr&PJtutjz_0J(GHRwtwwciTek-bIaBg%V#7c-HBT8R~>J#xU zgL=Tgo5kSwQ@{k&FDBYTu>nG&H6R4%mlj{5@Lzsq8)~ZwxTwdfbAlVL@YsNAV{#9tn>Iwgv;_RA#0-Y?yQuD?Cc^ zh9@1BYO{(&A2|zj*`WST7oc;B+{)ZYCwhM{V}Z|i%vIA^U!LH|c`1y(+HdgtQp;fS5j5!+S2Xz5I&=KgDk&MT=sr)^Sh3)fZW)=rB^tYgE@4IS{K zeviJjao*#UuBr}ICMTJY0|#daP9wqBM$vdiDj#2PdQi6ziVD+GQ9@Glhs47`$0`{m(9ZLR z#TPuR!fZkh>-F%_A(2s##1s>LU_2-fa05QYwW9@0adK!4d=VN4DNuAQDx*Rw>ji7B zW5+$S;*Xg0MTL*%r&`7sO+&u(~NGp9+771)Yv2Wr;J751U7#fnFAK6ToBhHM}Q^c}I za7@IE;Pb@-+^JxZ`A@`Xce5%zT=Ni6K%DJ&_idDer;!k>vc>-o!1o&kLS)P{HsXcQlDl zF-^Yk5EIH;BHa*e5G-}rJrNkPk-h1z@C7Qr^PQTyf%&a71GO_&t1CL|?!_glIr-B4 zZ#(Lzv)=lv_vYrxBh#C#S55B`()=lb^v01AkrV zUkOtJEBweXX>g@r%TS`^J-3rYRj5^R+O#52pPbz+Jc0Xc_%LSp5M5p|Nm3>Ch%2NSO51q!@V~NA@fAYNM;hIB#m<*GF!U_GtA zws}lk@n!SPJ9&Ej*Z1@o$Ob!igR}43?C7yXB459%x-nOG8(D=|@klzX5i#_^;jP>0 z9G}+<>ajbHQX`yG*lfM>mBS^`>^su2v5(BS4P5CugUvCQQSTU6#HOHbc^pX-O{gn76UEj}enpzo5_3 zPOtS&pD>~4=BeHz=yFb*Hj}SIE#{)3)z!B~bQ#yDLv1%t>Qj9kvuVzMeJ9MF2B~2- zCgeV6U6Loms)uyM?4(})z*>aQgBs}}an_D$?17oxl9uF@oO|Y=rC3uNc5&6r)k7&2m&+Lo`>&du zmmZ&%FuiEuJ^gEqJGyoDJ~n(x(X>0Y?0OgcpXjH$PsUm5j;M@ST6hE&=;Bg%o95Jg zu@>>PPT<@>DpHV_>f94YTq`zcnoM`b@_S*JET^To?31Y$Rg%ZqBy$QKZ~i``t`j`p zm={vZ7Y*pYxL3+^W5V3-PQwS^G=J_*brkB(=JSj}!&m3HBg2Y{maSNG<&|p|UR_ia z7U|AeJ$(2zq7hZoZ>x>G&|SlV{wL{`b7S$ecbmep*pPMC9e(7gEae?dJE~!*jha zUEV%CqUh{|In&L#2P#zRwrup))vLFTKI{5-FPnYtmTv75jdp{JFSuvv=I5&iw=)vk zb=z|8?91LoO^bQtTdK_KoJGmiKB$UsIq|xO1;?nzQD{O=|Ih<5+^Wmnm=sW!f9jg zAWm>J*QsIBZxDLifB!elT=v+0S?@b;x@mo{Q`=Q)V`b^}FTXk5^!lbnJ1;nI$I?wS zDSi7Z`u2@Z-;SbQkP(#>WneWuFPW~b?W$*yvC+plJj)ZswjI3Z>}VPpvZ`ucP+sli zIJTkWls&qbXq{~RT15WK!h+UG65^vPVk+Pyqg?*AxsuahXXmz0{c8HVpOms%(k{T# zlA0^;RT}(3f2WqB?7N!Zyzr7@Gri=JyI=k5_18zWZ}8sKp{k_$qmsTU-kRjfqN}cY z!|CC-47_a2tXXGXR(tXNzq@9dF~l8(l?T&^h~|*$keCQF_+#2Nzj;XOMCHrT_F$(@ zigd3jkOyN3Q+A> zIDNlmuSq|R9spJJ5Aj3};VT*TU;LyC$k^-%d(;i{y?0KHjq@~p>`sWCiVHy5*Jjt~%QVofv#WY`Ou8f7ztFMo2T6r@29QaQ5@xQj`QJ74*wYps_ykG~! zR*Yx=TB94&iwXf7VJ1v_A>u*EHQ$SwF=wt(a_zqxo!JI*tu{+SV>lAUAC7_&u=WIcYPK`~a&8Y>eaN0ShU&2h^{{lJt2M&*w&z*hT1Kl~DeL<4H0mu-Uh9z zYBnw3p%nRV$=Bur-7NAEbeU8Whb+(PPfU3P;xv; zNfq&B7HS;bRmg()GtT-4P5o1v+TC~O(h8vCE z#t7^jP)OPa)m#60o@l92>(Qm?RrD)P#%?ZjDGQcnI3_)?r)(ny(<2p}C}SAXL+PM= zhKdHzlY-U7_8ZqDs>m9yH#SCXD5GD-wMtmFOpU3Dt+AtRzuu{etR3M1GY|}g!(1dY z`d~K-xR#ElNlGqjxfI^xL8ueto1BaLxY65b*ElQ7J0(7?s9n4KG-Ke9l=Qs3oFUya z(<7~7+tt6S)2y-AfURliK?n#lun9Se>XJjWN^lcyAKQ2CN;F-le zii<0Hb(>JwEi$*HUHPb)Lq?oeIpLf>G2NrO<)4vy^}x)kc3o;5nuV?Hvt4?pVT#If zQWD~$+PUElTckNBHzh;YY08jhR9TKgjA=%LHKKE_KOalENp#W(*X-tRjZc?)i#K}9 zE-}_GY9DJGQ;hShuy)Hb)TUp?q(oQT@gvC?z+gkiI?@ywo))3B)OR3It{NR-SYolT z^I$j9c)B&yYO;WpsG2N5V@B7MICMD3XF%q3Mr9e-Xy@m`!J<%a2X75-*|LI5a^yLN zY6JQQSD7|gvf;F%@)WeFUp+XP18f8)oX2n!EtB-?8m27i6PDk_`^&0Xvj-0tTi~+m z!$;-R^fN}6M0YPN?A5WWk>2I3nw~u)>dlC=N6uVR>iubMPGql(3SE1T_wkIppi9T{ zsbkHn7G2+er&sxe}PxF?qddRhloF3dQ7`+c3MJM@SU`9O~xD-Xyb*lgf@ zG}SZ_hZ=MAh9Jbnfxs&F{ISrE-6$45nYtg?a%?w9f%D3Y_%iH=RkU2ms00}cX2H2C z!n5|c>sBM`t6Pl7f~sEWJ9087)lIu$X6Av<@6M^8P_((EX3o%tD@t>_c218f=$xD6-SIQ;Q%9|csLQ;?OZvML zzx^!3UU1?TD{XD_Jy&!X%w@X+YD?$eHn05Wzqxa!J?zAr*2bt*BXZIt4i!4QNqalP z;Z6F;*y{SuOgJ3YNJ2XnLkUr~*#moW%D4GX&Chm0PBF2^s<&l9sJdm20t{Q6SYK2# zTLE)|+T@P%i&~5eG9^&CfDR(j0yM!-+7m;V6kTye*o7GjyssMZ7kbxDiV6#Rs-S zn9vFT_~UjRjEMUBIB$dZsu(rPm5L<8dprUI^Yw4mR$OZ*b?uy4+hdu3`~>Qxhw^J( ziwp9zGvFJN6JnyV`(vStuTl^sC}yMbYOwPT7vrqhVCR?_nBjDifZ#ZyksjTaS1xEmVK( z{b}28u8%Qh&qmbIwt9X`w{ZGJuAS)gi%O?oAlJ$0n4HAsAV*NRQ9{?+?1cmlzp~V!El%~|vH zkaMbMbS=)#!0Ks1zUdB2PM~LKZD24B63ax{sB1pedB*%PikcjNW5#Ul8mb`RXzFU$ z=v?%j&Jmo6jO>=qp))7KT%Arawbar%S}CWI9pTD9vD+BrUAWwMba@A~)2h(Bjf=U| zjJneMb0gLJj4^MOckJBgxOU#d$3317bDF)^8hyQ=I*O|1_U3mwIP-0U@rZX^bgBuJ z=wRl}HvX9~X;PHhp}N{w8JVV5sZzXtn`x|Va?zDu0d1{NyNii&aJ-gTk&U`-lpZH{ z%*d%iT|hPyIQY;aqM?(x9b5b}ASlj}haTO~cgB7}2-E8jkstvH`QFf~Q9+aT5nb~7 zc`PHsm2mHOKlc)ELu>6+SB{J8_OYFrKD+lXpBdfLi;BJ4Z*~9ayIIt8; zs4%NKYo1k5o_xlczrC&hEn`jYd*H~S<&*P@hgY;W zI{xxRJ3PqX>VcV?_IB}hVDA%X;0R)L*Rou$w1Ei zJX_s?wE%Npn5&xGD4X;c@)1>L2OEK zq&u}(=|^iFy0sEp#!+A*wt@>0r|)#n&+oy)(qVa(1@6^Lo5C(y|H9qN7QK97YKQ5Mj9&jl-?ZW}1|w~j72>#`yBD~61lkefSV;5j4u z<(+qX!lb5H*Y9PzYhmrSMd|qAkV5DX2J7ALy;~)l0PkI`ph>=EjLx>n^>keAu zELS>ZcT9y$!O)S2727f~QAa8(^B<{A(v_L=EJ%do_1H*^`LwsakvS;&^*(R)3k!2^ zzTL7Sdp1?SkX%EFZfd?LBacgZo-_&~^76tW+=u?=Vv8#;FVg$+BXL+H3HB>jq(e8m zaMt62>X=TM9TQ^F8!RephgEGAnW&NNa!jqEdgf;IagNN{f?@f|45M5s+Gvh#a))1c z?nQ@e@Le6Ndj_M~jsHTt$<f_gbhu*l7nx>jXS0yHW{`Jm93%oNfj*bhT;vF^C?M|8POB_x~6B~8S zZ@4SYVjMYk`{TGIg{@j07(rby$LcJEAYkG!q&}2+S89=9bM2CN@c-P2!Js(*8XGmqx0 zw1%@k81u7CjQpzsAeOC+iAGur&Ly^iU3XRD zP~j=twQ3+fc<*w{H@1FUtkc-pM(E0fZ!EOV$yktVq(ynpF?%#e%uO&;qK$^;56vgL zW$tDfa`sulRmityw|k>IS$X5U?`??8Z2D^c$3uEuYbAB>+;#E={_l)MX6XOUG9#lp z0+XL%Wr04S)5#F5;)kp09>Si>OLRf@zjgV@shr7>5S=(8Z&IvKh z)v}MS7BT8HIj&kp#HDsI|Bdvjhn<{Mi;2zuM&Z~{@5E^>-A*si?#Bs(w$?RUPiqBQ zC*H4J7qG5LM+~%;f9mop5|<`hvDRfxzqbkkt@U#3prnSyNn`8*g^@kkjcRGTSEI)! z_;=MihYybMI*X=Z-sM$!gFw(C^uba>jOWciw+hmnSE5P?xz?xK%-)*MosM zIgZdW!EDH=EUKTEdX5h6!R?J`9oz>ajWbY^t1lj3IiT*0Ua^CE$EKR&tle;gPm{Y^YS7V<)j6u?X8bMudL zk7;_;Vwk@US|-5&J@1GWmliV7CZ3IS_j@BcLYvlF3a|#jxuC!i_7&(<&`KS2?NMkg zZyq?ZKDqf}KTWWb{WRg$!~*YAt}~ARA!ER%+KK`{Er?2o7If-eaWSRv>l*HHzCTwc zS5Hvx7z@3+4H{&V13M*1r5*iMwT+-3U^=HiTbEGu9cKxJZLTCWMcT4f2$o(AzqJ$_ z-D!)KRXL}lTUcmqY(DFx!uKnkcj>BiWBfXV!_S&N*}oK^SsjYze8uz|)=c*+3y!;T z)j1SO&z42obKq^FC7v@Coaf~BGf$qu?8JF&sCUpLH5WHJ>L#jB35%?}GGC=9Ptc$g zy-S3Xpb?m@jWAGwW{-ce?w?F&EoqHPNJz*}$mhBhPFKXn7R1M@eQFn{H0h>+RbW=K zt{6_H=4B9bd)S;-4cm+pf3$mLEX%RtqM9EwPc&66%Lq?36Pw*1pYR@UImMpgnSXqh zvHXkZBzwa&?_*EqoR~F*Po?cjy<d9X_t3ww;dtL`FDm_uuM#BAiln?oVqM(Wd9+ zo!s;EOR}}&mD*L_ZwF2oozff`T7GM2YQA@)`~7dTGR}Npcvbh-f_JK|Sm7@_Ynv66 zJyANyS=EzRop9V+&1}&5XeNyQFpscCv-wk6Bi!AQ6-Q{lGqU|^<1Sr9O!F=>7HXy2aqZFTyxq8=d2{K(Stk4RNN^E|$`JSZcFT7X7BvLf&SejcMy4y{rR!?So9R(h@nV^=75m!3J zWJ`^$H5XbeHcKfRR)9Vhu4mi6%Mx@c=0bALW+>CsVmL9#5;z(yEp##4UrL;=EsWvLi~*>eu7knz;5Cxm?{+yPQ4ja=y2H(?{N;+ONX_ z+0yf*&dKf4er9{?SjJ^9U2fdi^!)jWW@3!-p7*@UuueGxZkSmz=90P|6N;)MizEB? z3o4XhGv=NP7Cdvyn5I#l2aip#en0=wdB(2myw`1vO855d?T)Qmb6&q|FBzJV_*A%O zRA+DPXJfl|?U7*JTsg?-k=J9?wOh6N(5q|j`U~w6OOGEYA63V7bZA`!=VGX3OmhZX zW<1=eo9kpaVCBXhu{3gE%q_?QR4qJ`2E%Mfdo>%<9v2-vda4WAVK`y+79X|fZ_tz5 z+je_Z8L?*w<}u(7CY(mPrqvF^|Nm0L%!1YZdRLToSA*xVaS5?;`SG#bp#+hQ%?|Lx z&b}CZUX2EE?dhpouFVk9H9{}f?&fXAAyf_0`!<|)?y`*I-CVDF!@sBXM&3BH$HYQ^ z;jn^a#JT!S>q!@rog58*^biaKnMz`Jtr(q(a}2}3;b`Yt&wB3| zwV|P)d-S!L{qiQRn{vq$^GA8>jHvt>Y2`_!BbPn9zVCkso#ojUA&!*i!|N8BE@f+H^$SP+#b&$+NTkbB|vU2 z$xCj2Zhj`__4Qn>QV2W8mE+W6kVfP&*aO;;{)L&J26xZ*@10vlY`NW~`O>-fTu|}M zGdDNSFc%D8*t^q+*xK}jt^@zr*LW#VQzNfjUsrj9HMQyf8?P93QQxR&%+PkdXskKk zoY4FrMK&dorTj#0AA8m{=ccrq6Fj^d!^T+3 zp!pN8xv$Uog0AJ}ws!8tGZ(H%@jen8f6PH69Oa8jhxv98Y zH*-KlR8++1(GeW~HV;~NG|kQEnxWTZW_11H{JS?T)Y}&qZo2z?%6WwcC*b_!F2lau7zEt%FTz84DkNe#i2#enW=z z3tPV9`G;AvnQ{rDoL>d>OQ@0mP#G>Udj09#cCqoX?fMO`O^!G9^Ggc_Sy9?nX{RycX$)b ziq-U-&9Q)9PFY^BUT8XJ6!k1FPWB#IY*Z$6B-Mw#ql}8Is&sZn_eEiTJ~XeWkg|so z!eH53YF3zAfKjD%_Y_& z-n+;rK}xH;MMoi55GSYBI`rG>4Avoo$1n8N?=rM^v!1qdn4SwfNp^NtE)4TI89nnK zJna43ZO?e`_WDYDjyK*k<{9&i7n@6r1>ReX+%Ba9^J6lG4eauhRjd5p>utyPs=b*A z)hFXFLeAjjaNz3&_5 z^4DHpZkXP7#(MAh8OD?sUhv-U&(9u)0a_n1+skzSd890r6Wh*deSd#+^3;pgW;j{G zyku(-=8Q(tICYrG)iKf;4Q1!4}MoYze5W`Hv;0mj6yGv6$0HOwDkgR$}p|kyzsCCYE#>iKU!wVjX@o zvD5g%7DL_aR>X;L_8^Ld`N^cIARf}B{19ZI0%axGfedT&Uv^J;XHR)?Txs;v6W-dF zT$aQFrCN!?aHID1hq}ad@a}qTYkWp>pl^rvp(O6RWL{|LLue2&lb+0@(KU3>x1|rM zG6H=l5EK(fD04cS!z}>=edq@h3-qBMPb|=felW2>ANujc0)6NQ6ASdAA5SdMhkh`z zKp*e%ZzY$IkGtq-Ali&f<-LMfoiTGrwUS1m7S`+3^fJ84QEKAT4bleedpoZ z^oVX{NOTN{H3lg0X~eJYtx-{?RfQM5eed++G8R)s`t{+`Cd7)FD4oT$4>=6&a4&xQ z&W?<>lPbF;>Uz{aI9Zw||M2S*?^&N`oUv$R=HQ|}x7T~xdlsFO-h1wd@z)LAFOj5CN*4$;=j#$NZ0PG0eHp?}8DXR2gfdJ!Gkl^c2_xM1$`&IYR)Ao)sF67X^1MnS6$;1n#M5RHvv?e}0p2)4M**v4b zh!0uzhnZ)4GmJfxy_FaHOVX7br#0W3VLUg*TWPqvWvX{(Hxh}qGt8G<`vYa^!%*-G zybwJ2B@m%YA3qgtEkw%_NR^MYLXe|m`|YLpW3hh;RL z-BJcjAEU7x<5m-8;mYsozG;@9;R%l9&_V>cwjbb$0&7f%Ckp9W!Nq?bi_F;UXv;v| zZf|4r?kmS@Ar~3sw?uKKJfb|bFk0`Wra?<>X;)CT+!Fk zGRnR>yD!YCW&KRVNUGtf_F?=dE!28U&vF$x>0zgSW`?mmqx~5*gC}&TwTgQdcCP50 zo{!y_^ida;k19_uNawGpXHnA^=A+Fc&HN5!Ikml`lHCdEiK!_?TxSuJoSK##otBf^ zIX*eMI4!kfVpe8;Zd`mqVsf&xj+J`kXj5JPrLE500#(($5iT)Slz6pFxzw)TDrP`x zpiFNXXCXoaP1@MHS~o&^9D++nmg2w7x!#QQ&dw<<%>l;GIwdD}Qe?R8Z<}0IjxTRc zbaYM77snsue`j2rTt2<|ARtAcoJu>KeQ#!i@5(m*)1|b&4wkZMW@ssqyPQ;q|C?nr z9}1SS`GJ4CjH>X?@&EsHJsRf)%jo^`hsp>I47N2rL3eBCqA#Fo5tG$D$%@VQ^u|!K zlXtqkF(X*K-l3&a8a?6V`M z{U0o)wWGC^ZR_y=N*T=y|IIS$Sm9*M|DUaG*nT$|{aqOSHmCnRt`JhEEuN;*l0j4am zYUP0GBs)i$2xgS==7Q!!$(@YVI$d}NSq82%Kr%WdyVCt*M`sagBcv>;(zlvZ4FNgy zk4zxWz{LWNr*%>66sM&uCs7m%|`W+Kh>Pb%Xh*aYCPrG zQ%h8?4?mJ=u@*P`6kCisKH0-YRBR`@5}hoA=L-r@ocAB6&cPL0 zsS7?`<*P2Xe~qDPdO@3cB8lS{+s#xd-#v$@BjYR>@D93t1-|AJZQ|Z-%OG8e2-%r8GUr@ z!}FCe4xvje=X8Oy=;YFH7QkU0!aKCA`GV$Zbj_z$dA`#ryvvNa-aFn3=^9p}c~$e} z8MRl1b`Hwsoz&7df@M3YhjJYRg*8zrVrui>GNgO@Zodsi)!A#)S%YOO;NZ*2Y#fBYY|!5fM36FGiqQ{NPhp`H9M`U2NdX;42XlbDmhc6rZyP$s#) zk}K_%vc&y?40LUfdSjYQLg2Iv)Dm`nm>74OePy6|vrNIg+&WAeD`gDW2xbJJzo!Cq zuL!tfG`z;uG=i``lJh*5%X(J>226rTsQOkEArbcHP#3G?udVX zsqmQ6$F-N=Z%Ms%qHXe!n2uu1BQ5Yn&u_E|vsW9N^52eW%o0vkdH|)8w-# z|Ihet4D*;T@eOf&-y%u39>={(YWc2ObEEXJ%lN%PGR^t?p5U8i*GjrMLwdSjk|O&$ zFopNaq}XmCPPTNjx)E-tRMKGvc%4o-u6SZ$=8x; zKP^SpUMVymmkRSe>ETW%y}kZ&k=Thno+OQdz3RDeQy?@QW8 zTk48@JzlAI-%J|xU+vx2CfatBTy7kv&mNb{owAIOI<0r7?W1Yq4`hsSR0=_cD~z^n z0(`%*QtC90#t%=R4YV(6TLfHv*8T@95DiN7+nv7GS%y2$+TQ|j+Cux*zCiz)MFBr% zSbhTDlrh%%xR=X^+W(xk?LgiBO3KW?OR4#&R5|tZ1;2R~X}Sm8X~+4#ZQBn*exo!^ zUBp2{+w_(H=x@r|l3wWZzmTW<{nj+x=hL@VgLCN*w}AV2Zs)wuA)g%cK1tKDXx|a2 zCw=~Z0H(GbXsN3>^j7|bI{KHww$_K{nXltYVeN({2A)G*$G-<*=-bvxiL_^7>N7}# z{)b(6lx#m$-kw|KLiG>Z0S+-1e1`0j?e`!q%UwzMJ>q+a_dB8UwdSi*sjv%p-o-Pp zUZ-uZl@i=`_Hs!!{~@Ug=A90lLFi#^lwyTGqqoxEykE){I=5MQ(jC9P{4UeqA;__M zFo)Q**$vRS`SkZ@-#fhT5x_uuGtV0Deu;NYf%YztZ1+P_rr;DrJ@qlren7fdNzzH7 z&#sTi^H~{TF<+R=p({(Ihq};{ed>2An3wX6fp@kCx*80p^Npr6Ps-FKzn1vw7xZrl zuVH>ln9Y>oA>v&~y14q=nd6!7*qiBZ=g=2*&i_8tI?F5Py}i*~)grrXCge)!Uy)efS_5sE8=`!~QMv=@IIXc~{d6yIV#ng3s9klr-r~Y;Z_>j0zeuU?WHos5eSN_#^%71)c9L9(!Lf;3=r)5-so4JPX zuC*hm$4&HM`hYos@rnz2m{TNAU2R|7OtVfpSjTt<<~qXPz%xLBqEZq3ZM`ZJ=nE5+ zN3g=AJ9MF^KL0NuOY?S5^nIvs^1njXb7*0`mdiDQwhaJ&wI0;k({Mxafi!s9NrUpe z#!l&?ZYz9$;5kY8rWW1_{>gKH2sZE>fUAOV0eA}Gv$jbObz9TpZtjh~L)3MdIYvqp zZJ+hGRmS@a|2u7$Hb~@mqWL!OZ=>E>FF+U0w8!}>{E)tOZ+P+ZY1;>&uRK#n6^>tT z_|gE(XKDZU1zqf>7ae|IdHz!)8h^3Nd`m6ZaHoathCFAWY>1P|^ov$NIlT~x#yEd1!EXE&kssZT{~LuF)nkY8;Z}x`uoRVb(I2z`vK@> zAbeN~>AXeUt<>#TzFOBn`2CKQ<8AW5bwAabu4lggIjo-y^t{6J%YK~n{50>L#Gmz1 zVC`0`u5p``Iry3MFn~yNxgunJ*Ms$)f^{B}wv*tj{dE2JnjbQV_rUj$lR@9oetQyp zZxIHx^>=<(P^LWIRkVE{EN_T^XjpwNvY(|7zAd$$A>G(+b^xr!e18wb(KVSu*Jw3%Khke~=lJ0t zNjj;5}7;ApxPTz(g^mhwZ@@|uF zW6SSg*j255hCTKJ8OQg)#9xta5BD9m>lo|)unD*x*4h;c^99-t1b;K{R`G1wWBl*c z-?d(1tY1l@LZ5@MM#I|-B-}vg7_y!pM7HnqkoN;zUph3ehbx_UiURO=WI_7eJK)E& zCtKn|q4FGs@o)f}nEL{L<_zEat?-8nd3S(!Kzdrsc}ILFog1V8S7?_C$3@;`d_ljP zf$dq%hxf>d63r&xo92VQkBq&(W9qg%6VGb!r{&CRJDA&^k|Zk&_j&TlmoCuupMYHW z&Ytj~UF_G8U6n{i(6ze2+uY>H;3lcO%zTFaaV=>MBk$MA>ssG2^*)L_7Sz$dU-PCm-hqplHXWs-3zgN2Gb10PFhy6_Yt6xM3ayJeC5^|yp z=Gp6|%8Ev|bewreWp+ObpI7I1*Ix*`#aCxV`fgB{cm+WO(kgI$>-gQG{_5viM1Stl znuq(Zr1d&;M_1{k(C4;bO`dN*!n_(qe3fbb0I0WfTzl#Fdwl~4*VFYmJkKN0&nG3% zb;Q@xb)0pnglQ=pfBg^v6M-WK_sy6_0MsXr(T zxcdA*2VMIF@^i{Wxt!m~F5q1Y?LRTT9+OhLhO)m$|JSf9r8Ba?!!G6-=l2@c!0qTq zrvQDe)_y~}YCp5@@qJ_uVl92M3}!B>^;r0I41jda{nT;5Rau|)68zaqq#KX{{;#^v z$q>!ot?l~)ya~^q0sMAJ6_zyWPl@UHtL&6-_H<<}_&Rj+*D_pTMbJ+bcf*HX%v_DE z)}DdelQ41cJIK1tub?3@xX0iFRrcKq;W@_t+xJL||9U5k`p3B5ky2Nz?>2WFetqfl zzh&N@gU(_D_53z{pLMnM3hOI?o!(mLSgFGPh;nRTelwZt-h{RRmHUSv$Jqy%z&f{# zdRW7m`bQ}DdCr~uCXaf?)EF~Nav3vw%_zyi{m9u1$oB2@ z;A?$v#z|)Gv9V&%s&HNW&h+f5aH?p{pmqp=tjAo){Q69dylqK zD1AG^9D6pjn!O*_5`K4s579h!N*-=!7iRr{j zzTHUcG-1ZlNBm*c?PPWHHQ{!l{St8t&5o@9^nIxeF_9A(2jH9Ep$z1$zs)y&pDLIa zS5w#HtxITg-A6io3?Zz56emtM8O^s}2F|jN$=Sxc@UJgu`!&0$gJh5cDnNF?)#sC7 z>|wrg{CvMNclvgY-7j+ZhZdK8l?uxPKZfi2 zD|7Zs{C-}B)1TfV-AMCgzWsCLgLhDmA+G%o__pm*;=axv&0gldp`edUHvdT*n$9@7*bd#R`p-NU z2hvoxSo6`}uCDa|Q^36oI&}|pM_M+L(9I0=3| z*V$J{Fh3%l&4jbr%lnc#x)~k=z&9pXOQ^f4v^&qCu)iWt;I3xx_Lw9@Y?OrdJTsPi zYB=QIk*>>-xlJG*U=5T2kNHpLrZ|26Ufk?|2Q7~cjl;Rn0j$AmNvoFqkV@SzMwS@o zj`8mu>+iM*J4+JP&;1iCA>ySA=-r6h!XL0)}*Cm62@ z))Z(raQ3Qozd970G1K3yPZJ`h@T_~v>}#^dN^$Tsb;<943s)0gq3OWinJ+IBi{cK8g>jq-w=+}!$<#ydHQUgRYzc@ zj9{;^rvu~&@NFZ5zggQ%;km)VI^L~w-Z{S+U(@mL@1QlF^P6u_9!=Ap(Gunl#Q%d6 z-w9(g&j;T7^ZtVOUxeg6+!>FmbJ7Cd_i@mgpYz)u3N62wy1SCK{`Y;$CiVuWQ}1`l znS`luu!}r*Ie9w2;b-)FPhA%`$c5kg?mqIri*F`a)l!$It() zzk}$g7zgA+<3qXFajVdQcv@};YZdky^nPmOE(h6i zo2eKhm$P4{x;mE|M~JI<4E`VvI~meE2ILyY(X)6A-fEoV$LgawHV~$drMei?sP_n& zWSosmE#NwO7OHCzfPJehLl$U%2m!Tk1+!gxzo zSmP-dFw3RH&NQO^kaFn3L)Jf#>EnjQsUC`%s$f2O*Y%40B%mwv5bk$iK5KL_ek=DY z*n5p;-%4DY(H)T{IBh?yJ7gGh=#(Jn>!p|>(8)$TfAF2q^!1&=eusj1&0scj z=1}4*l4K~dpkZb*V|BPpVGS`%VQz(12Vj-+?iR^V!269i;F(F|zx-}5x(2(wVgu_B+ z)4d9F7XASQf8)OL?;@C{x}-HnVXu}9+>m^IcbNP9`3L+j!DR__d}AQqzg-T}L)+>5 z5v(J^tOC>hIDTpCbCd4zsO;@N^IIw9c_Fwk;Pz9!vp_%O63r7Q{BXi(A5q*M!1w%a zK8HAjBg4-0VOSL{%5bo{Zv+-Z*=g<?yv(uBn=){%*pE&WSwwZ>cj*hQhOh;5U07HLfSjS_Qfs5`=M|&nx16;)ii!y+3aA zOv*jeNN)WO!{qs4-w4K&*7^GZ(0@)mN!SHunM^f%2c1ZtH1wa{!SZc zJA^{prVXq@=7{fNF@7h(d<&kZ4Nehn3Slm83&LI87AnJYZI^~1-20Ky#>?dj>n%Ck z7TL+Vun?ZCj(%3GzeAt-oq$aEYWf{Lt!V~wU)2{4{#L&qq_z4!7~b=!G%EBr>Glo6 z^RbLk;17(cU`hZ}U6&$r0rno&Q+s4h06FTqS4aaW44{Fv?NZj%=NX?Mv*bC5ll%rM zy=aO89pKfTqvCR)wF!_#Aho|Gi4? zsHX^H9CUIVe!q|gAexr)r=T2^fu6Rmx%OexZl4U{R#zLoX|lB*y1QPkMaMbj{~}!1 z%CAmUuCP*xXRkQkvjBO>HsmG&XjoUaj5<}E{|kPn`X0(V!2K7#Aq4hf8PEoK{LZsa z_zu~(pts^h=VdW-;9xKWJ(MAmWFMAjcOJ4R%#*4ch+m3)s>Z$>-J>7F+ob6aQ-4y} z^~4Ru9qcP)vYsC=Bi*z7&-(j&kcEGi>zBSyaYyMhdTeKgpM{Pe(BHvl-&@!gWNKk{kW8_J=TxPnL%Pi{zYYqp#Gp!T8GqFZfrto~^ zM?`~^Dcb$T56{>51M4I9wiMR}+>elJ2jEJl?oI~-eIL6NgtdKg=4r)AhpzsHGc4QD zIng|p(k?%SY3LM>l1>`VZPK=Z@Sclguywsu+b>g3$bN05J1QsEa6M&^Q_R z^_}S&<(rI7#~H3hFp7NYa2JyX@{6z!aQo6HP66lLO#15oRXiac*Fd&65+x;YVkKRB=_OIYI&UJD3t;X?eCGK0$+@a8=*C^v$_Q6gc3Bjx22;UPh0Bm4>;5DDSI8e8SnW25YX#PcXyM1=o6%T2e*-q`oD^M7wKif z1BSTY_Z$qT??ceRzRUM;$nO;UcG6Y6LBD84ua@U3=Ffit%qtVnc_?vkCUNx{-3fT& zk#-*Q9Y8KQ(&insc$>9UM%XEY>5Cs=(Z3wo;>`}?rxxqwSTho;m1++~$BYJ3gPi$+w4nI2Sy1JbMy_3UeO&WVjj^xoshPi-p83 zM7EefnFxsKmA@a{o$5T{A0u@jw>jp#dW^ljmNqTO3MnD4V^ zw3|3hjPHeF*c-TP#;@~ixV}P{S<#Vw-}#&$yp3xg*^4vp<#_>D#P{|!8Pk2AqX$}z zKIj$bd|qKc<~z)uO`?TWV%I+Sn_9jJkVdXKLz0afBpiLf4;}YG-(GW=grk43*Y1z* zzG5p_fDX$`($4)7`sNLk=>qOGGts@gj7qxihh07R z-dO4m9SY#M#sT)Y-yyv;%DxAk#+|;OGM3u2kM2SSXqg{KJp16!o1go>WnVg;Gdo|| zeSBZrhtY3&6aB}h;1|5UFI?k%U%MvwzV-*Y#+IQ4( z@AVyVai6(4j(pCeERRs8MaOuIrqXE_(27-_tgg9r!ar!`;c|^D0R(GPlh#zdIQL~-b2P%h5Rg< z^uws*Xyj+n25pFZqy7{akKai^<}to4kYRLIAn!WiQ+wB(>-W1XPW}IoTSzF_EpS$` z-I2s^WA%Cz?V3&CeO9mdc^X;qrh&DfiPbB9;S0fTY+)?LY+#H?WC|9Ik)pOgA`*o} zD0-Vn%wmjJ%oB-YE+Jg_C5Ub4xw7XyMhBEMdaU^@STrwyTFSgC4}ozBGQ$mde=Ru z8R6HRINg_mcfk=Zi{pKdYLU`)B4um1CzsznrwXc;ph={1gh=msB4?0ypT#0ogy~z! z_Sp)N{`mJNjcU>!&?qu+oXDUUKw33~8=MQsr*@0Tkb03i(jAIBl(@qbgc-IC><5QM zh7)&qIhX`^Kb&-iZv~`3{9SNVWJDw&-4Rt_9GC}IfsJ50co7@~O_1CKPy%YeRInJV z0h_@tuwUfN72qh_hjk*Osh2UtA4|NklzDuX$iy_d3FVp0_fOskDDRZ+B2yQNoQ>bK zd7{_s%JivVr^t*wA~O%M_4o>S9~==mhw?P66FHak&kF-7BC~lvhjPqaC31cYs0Z6b zE+CJ2q&aT|<^m!Czc0)UpbnRJ7p@gq)F5&ZVJ;%hMF&MLrv4XK0m5HG`j>1Lxzq+} zpc=d)vSho+(gZ*`E+hQ%10pLZw;Bk>zat8oTc1-5_}!4Z*_#9vtn zrUKGfr68|W&w;~;_F}+JK=^A$0MfW-G1x6~Z4H0Hk@+%QjMO|{6) zxqx?TZG_v0L~bL^b;MmqoOQ=UZs+~&l5wcgCAGMYa(4fkPs{+AZ=>q{wf& zi)>rLW(e;eNfUXL-`h8VZQwcZKA>E`C9U7`?zdCHVnCSR?gR%xlgJM8*-;H9frVf# z*agVzcL{)ZzoU%5TLrd(cfnCaz?5NUE~o-?KqJ@+_JAWIj}iB=5;h)4_i@sFd^I5N z$I1KmgxfV$ih|AnjKdgDv1ifd8NI|8q6K|IhgU`CXCMVn7X; z12%$JKoc7R)Y1M*Fcma{?cjjO>ydyoUT+Y2gM8mu1&H?s@82MwH+PG?RVeZ{;oiv- zd3T1$d%XY45s|-w55hz~tP=UC97`r?7)Idt$0;KJwM*pCL6J}S{b`BF-$?s2opl*Rz!mXZ$hqESw~o|L^8s37l3+}>H>uo!3X+$Rk*iP1L_Y!Rd13NiX8h*8b& z0Rpyz7sVJK1k#>(08=7+#9)3hrpyrIEYg}v8fVvwF>Ra}(-lj_m_fK1q%(uK zGl@5AD|iJQ72})~K%8@kdk$$d>=xs^LNR8O{~Yo;pLZAFH=lGCRf};^qZk)Y0&~Pz z%=cbG9+$2bW62sZmTnf~GU6^L?JLOR3d(lHK{h<8qiZY0xM7AEx9k$*_NijrwNs3{ z!960zrj26UTf-*7Dls;%72`gBLnDn}ju2x@49EhM`vLNL0RIPg|G;i=02~qH!7y-0 zj9={+V=Ml@Chv#liSe5?Vr<(W#>157(KJvFNaIo7J&ON!(%FIkjum43Zj%^08^w4$ zQjFi@KEdy&Mu-93F?RDEPtOtK8Qwj!U5sZN0CApO2X+I>vBw6awWkiu0BgWzK-@j= zf}>*WCH~%APz5N*-bSz$5N0o7_7dg~DWE$boqf3b)FtodCV_=wJdgW4?jKi)@gixx zSPl*X;=Z&=j6c-_^8eFPF0C^tX3Gn|M|IbSR`F}nK5Cd=?%mGUQ@4`2LEr9Uw{$?a$A}hsg ze;7208AbZhq#e6U%*2CYCa)1QWgJ*4W{1V7HSye$eA4-yxmL`qO=4!(0pfN_0Hl>u zC1zf=nEAXfcwfw-?qYW4ckzp2cA*U2YQ%(}H+yUVcqUXP|PKB#9Ue_=CZ9~UPf6~P_E1OhQJ8)FYw`{=oytP%#w?}}ffN*b?-J(SsbC&h1vZNL9_4#)H#h)}i20W=K-j-jiuu2#5{CR%uknswSYW6eGa@2c=xvr zVty704v6_V>3)vi7vu}gGr!m=CUnjGJ8Au$e2(BA*#`Cl!hM+rszC!-4aoaT(*4Q? zg<}3A0h9pZ{bMRvEap+dA3Z4MKbMO6HTiu**l$SZ8}j*P9GC}If$d@*dYf3WJH?8hBUVzbSSf_-$h(Ycu`;oBpOpp6|st^f=01Q8pP_dO02F+#pYDkZ)0NU?gpD^>+@DoLx4h*kBX zSp6!+8o)FBnKf{eSc6DsP$Sq3c7gqX@Pn(xswJ%QYcR)BSYbgtMW)|I4vWi{9;R%0a~4C{TXk@B$Kw^rh}aw;J2 zm883p?^(GEkmgT#|C16>Bi2>qy($cB6zl3nv92NCYsqUh;jbg@>z9gk1MhEGDApSC zypgy!)rocUII-5^cgt$AZe1(ZZS`WUiv*;zp7-l#h;_$0u{KnRb!VYicdrubXO#CI z(%7^~tj#H6-M3w=UlIrY(0X8-SPzo#ugHHZ@wSrxuSw$}-ao|e-|P}=+X2ud7PQfN znD2X-JRc^`!@Pf(vOaQ5EcirgJ9+%JN~|4}jkU4G`qz4l^nOpCyTZWxfUr-*i1j4t z>@ER>e|o7{&j9G7^=y+^dlCTe;P0$`Pka;nnZ+8^Vx4KdhTs0J zV!ggatT*O}_2ve#4y*;de`~Q=&_nC(MnE~Bh1NTy^&akDbH#c;ORNtl^M^CUI=EY` zk6#q)69I?CI7bc_gj?W@I(suw$Ys@O4vkJ%1h6g!T0@#SC=SSWU479iimIbtV~R}$$bHHn?fyJYf* z2H7b~!CJ5j>=(O(fD}Ny4y4~2Xc&@75|S{4AqpXc z5JDKj@A2AsA6uD=@8|o+@Amt(+r9Veyk5`O^Ywf^U)Sfnch(eg%~lg@jK#;y?y~-foERhWKu~h+*Dk z-BSVLyRQV$)BOOk^dx|C(iZ^OkiHWfBzAH9*{c)ay?;3FUa@ma!&n7kmX+x6;-H!0( zA!5VRiH(4sktqKR$d5_{`-z_gea z6k?O|h~*&;b2gitKrDY5u_=(B+7v+7wB5uC%853z2W$W?~nCi;38ROkxX5h+TrR7D4{9p2RL+Ol)x_u`3a` zB#GG4JYrYp61!G`eZ;PVp6iwpyB={jq!TN`=Z%Yr-87om%^Ads5iTw!wj6fc0=X5) zzj6VwTXzw=9d?u?61#)IYGSJpzY4Oe)`AMK8&na)y-#*$637H&h^<~s>@GtYvAc_i zt;r@icRf0_VN0pvcN4WR4O zQcwYq|5N1s44->96Z^c3*cX+=zAPcOZyvF)vWa~?k{I@oY(L7x{LQ{Y_po%!69MwzcRAR>m0&a2N$fWUVB2r7?e}70f0Pra z4aBY8#GRgGzNN!?8F86LTx}(;Gr=a}F4EjwuoxiStpxa1k4K;(I6&O@!Diw?G4Uf( z!AP*5c)bd+lX%=x;z!1TBI5N&6F;gcK)$256F(-Mc*A+bkKIT7IK(BC5^tPA{Deis z6Som>0@;%gZdyXTSt{5{ymMOBHju*TO%zQdfFhpT^jKYMZ`NIJvD`R zC)nDlDe=zn0BM~e-x+D0VOQtXU=#5!+li;4ysoH6H-x*Rj_I)f6y!MtaXnWO?^RA5 z=W3n-`Mw8<_v=YKa|3alsrl(i0Qviu0L1mL1kjO1pdo-AS))NAfJ_!-v#N*>fL#N! zK^|B@eBds^7nT6>3__kknP3c?0G2GBQb1E>TjbHpOzXCx6H zl~4RE1#LD?tVE@vtj55kL?2uzUh+n1C{{f8`UPXTmbD z73?KGF#%))lrs_X6Za9H6bGO)Zvj{hHi7K`Y3DM4JUGkq$p}y0Mm&Ej@hK>0D$2op z%BP~7sVHaaBI47=5HCmuu&ZDv@$+*)CGqL_oDP}kkeR*`loOw^ns{Lr*iC#U%DVu0 zW+8p{Y~pj00LqwifcV^akPaYsVIkO0eBMFg7eW90O~fzWMts3q;tQew66m`Wc`w~d z{4yUP{W7Fqj{KLyj?1C%3dCQrmH6VN#IGzReigz?pmzzbmz06+#Fv&3zq%(to@>Ch z(6`J`MI7@szdjM9fozZm7Jy=~7E}3O&Oj;gN3y^H;_H3T6r=;Bug?dI0OZ$~gKc0BI7l4NzWAfb#5ZISe+;r4alH{X zZG?@FLuS)F;!h+Jf0BW%#GhJ39Q%6ybO!M)BZ)t=fcVxj;?Kg)=TV2}cM-3EEft%I zzldwB1$k&oCYmK>YVb0CxWlfBarf zf*433VHJ|F_mW^4B(Ucf0$~*g4v^4gB-|_#-dYkqu8$zlltjIyB;pWvWC@A-1tg9_ z-lOm>r1#ZzYk8^yD;94v?=6!fi4^C5g7H0pi=0 z0hE(c0-&os;yUan(Xom|r$mrKqBC@NhHmV+MHlGBo?CQdpb#Lwdpt-5(A9ktiFD*S z8D;e-B5_J7iC%GF1E?U;8`r(@*=IIbOd=zLMBhCm`Vm0g`$0!0(lU!loYoU$lQ?JW2;h}56 zW)eBDe^@5iM`Ad_XEY@-DvQME4J6J+zOhLp#z9YRIf;qeN#r5VdAmqVArjLnNK8lC zOz4}lm&AO87bcUq6m_^9HY`3s;>rvXOL~%6ngbwnHR7*H0;tEeh`$zgEK3Aw0QJ8P zdas)ewu3$3Ac^Y}z*2zp8yF}6$X^6Km`g=bDTy0>fa@C#wM^GK=1Z^0J-fI0CI0a?#(Q)mc(1}0J1xv zdk5rqrh+0+3J|}ulEmAif%)7|;+;5<0xdW0P*kcCb5eE z>c0#6cVzR5o zHiJqMAE7UN1V4Q=8x(^LU?->|@o__t21bI}pcrfbI{|d=i39N29+b7G2$X{g5;!M` zPZ0kJWIu(>r;7mcdp>hM(w z*bH`oDiU8W21v*Kc!6h*;+r&p>u>gw_!eb;i}df30POekAa?GDcA|BNc!2Jlw<%~f>f}UMLc9U$B309MAjC}t{9?~mGo&dQM zvcWQd>qL~B2)&80@kH2vBJ`aI-<*iFCa|MPE+_@4%SkB!<({;aWK#vQ0Q5B7PO@1; zfHInuf&C<#r-IpF6F_=%Zczk~uLbnB*aZ%dO!7ffkPdP{ zK3D`+f^x78>;VT!wu}eKAOnmBgq=Oug4;F!ypd4%id%!`G$?+f=WPs5CvN-F@@JU9PA<4-Uq2* zG(dd&GLjv3kxWe>*$MHT3rKcByLQ=1G7bGKZ4=3^XxpwO0QtM2tZrz>ZqV1Qie&d( zfc8z#1dvbPNfP@*dGa!VaF2KZy*+Y3AwZcuia-^~QyK!uo-z_tlI#iHJ&Q^9f{ne* zXBElb$k%%($v#;CeKKPtK>iHK_k~4`$;)qDL|bELgzr_8N@&~C?=T=o!RLiA3!b}y0iC@9Gn0UH+UsL+~8d#hxh>c zhU9^Au!rPO=owl;GA9mz*4Z6 zJuT!MT{p=0Sbu#4o?kh{7m$OQRd z8Q2P-=Njm_26?VQo@-FnH7M&E6A0%0XJVhx0HW$qXDC0)R-Hbdp$AdJ0>*8!s2#Ucb0NZd5 zk+yh?oLx8-GW|7>0{2MArZbUg75&w7!$OMHXHzfh+*pyH5iM1r3+)whU zc#sB0g9TtU*a;4j+>E@NbHHMN@;2`#`7{HNe;V>nL;h*VKV1R#k=zmoGQe!G5^M&0 zNIs)LGROw7 zUR_4AGLPhTpX8f{6q0YHk=y~f9mul-`gTBO2lVWy0DDR9R3HiT1S3hljeKt--`l%M zzEei>-K8YoBa-h!&jxqlnUZ^3u7Nq&zq4=9p9q?7y+@;~k)S(Qfe=TwrvWRd(egXBS6{|0cM zN&b;fiW0$I{H77ZZ_g=GEDL|ZYYZt-jK6;6g9MNavOys*@teSIQVMxg5IlTw!=IY0hq!tuuiiGW zpHy5aKp97t0O+a@9YAQPaRjwPg0^GS6|A=MdryFf=8@^)=V3iq&7 zcgS_GB9)G^)Ay4)8M;s21`x*nMD;+PQ&LIw97C!X^!82!6<{B!KFHGtX?;cll+_1i z_1OyclFIM_=?Mw}${T`wL-vyz3i+Xs9}4-QkRJ-2Lm{67 z`J8M3`J7S!{llPt803ecF2m-5wO~7_A~hWH!y!K$^4P1W;Tr(*jKJqe=*FH!jY=kU zrULjJ9SWcUNlgJ$!L&+J=cByoJxR@AqzXrpnwbgkdBGS`vsROu zokwa8^v)>;C}YlUQgb0YHwnOwxdi}nb1O*AOCWU-%A5}!7iW-K0I-Iug?mU{g1nb( zBX#LKQj4~dx?%yT#mIkULx4P2LeG^&U<24k>Z&+^yjP+8tCoVbq?W8CwG{cU#^=@L zq^?;^3Tv=hhPdleKn|Dd_6JlIa^CZrcbZh0oCB6Uj{ zsTIYfZiTMfHj%o$gj5Ogl@tQRmmscWH>o=qNCH`)02F}=u#eO#l(8xUz>ZbWw+ixi zqP#oPz-Usd5x*Mtt=0c0MB%qGZeN(0cpX#rRbD!@Kc zPs9O~|HMczkJOXU|5QU#n-c)ipI%IA3*xpwC)QN8wHRz7h5Lf)xt*jcU`xebQZH-< zu;E2qznBcNK_P(rOQobf!(CuUPcP*sCpOI?^cq+y)(57WxfZ!@8to6-_HT~#M-Jp*h=a{l=l(BSUc5c zi%ETs>n|3N`ZAZ)zLBK9Dj@ZBIjL{xO~9j>v~sUINwV>+oSeEyt4>X)UY zeqBWBAU=PC-0zj7{wN_$J4svH@LSOgY$eThlNL)!V=dDPpSqN^mqFT(2YW~ddr8+@ zP5Q_<()AmXKB_6{_#D#5Z6$sD0n!ObZ_Gdy=|lxk#))G9(wdZ!J}HlM(`?esW|PMK z1l=NwbW#fGmWXS$opfu+C4;um-*y-2cCaU9E$Q|>Nq5L0jdeq(#)E^TJ0sjBm2}z| z(p|Tc?p{tB=T_YVpQj*iFWAu=X?+%x?pHuM6M9a=C+2p2`cBgQVNVu52O@6JCeqml zNDqOYq1#C3AbuDa(UbH@aE4EMR728d#*rQko6kxneRc-va|%h1DJ4CQNRMwyIv3?l zC?Y*Ejr62M(s|H*?mW_yQ%L6{ZVGfwIY@dc%A2;8bio4B=NFTnUPgKb@)d%aOG)FL zq-R0btSZuT5=hTQc^77qo;RBGMTnn|axX@`79j7!64ICCkzR!QF4|1`as|psU$L9? z;tJANL2k)5(m0D~%xN0)nZ6dbFC(y@^!2zdf^9b2!g=qfHFy*wR2=PgLP zWjpB=kXr#AEB29IIT9f4)+Df&^lgb?6Y1L#fBQkwC8*1)ouuzdAbs~P(rYpa-3cru zeNP-HA${*`()Tq4$X7Oo^!+IJ{?()(KzR@BCH){|*TNp``SnAi!G6-~7Lk4!c^*do za>$n#fCHo-fv!iYNUtv>{b(G3{TrZX!#iXV)tgSQ;4PyIs2PctE!g%lcpE8_|quXNf=K17?SXfdfZI6W& z#ajunu(k$TDY39ibM5)Du!nafRK>zR-rENM^NWxFNR@MAfb88X3L@l z&13peOS}2Xn2bXWn#XLj805=i@to#0uZ)H9*Ud2^tLtk?s^4N@ER4Uq(IQr^r75ok zzTIhjWx0^YH%6<&9>ur#Iu`b+L6RK{2Q)XSVJv(Ev83x`;d+#ibWu>TVE23Z$>s|y-=n(CA5z9L#qY($@OFLNi;gSaV*@D z+~m2jaI0E&wkDPwwM#N(Bu8~=gPnO)_qG(59P5WxJ4#I66VsDI=e1$6aC@@bOpS#* zP<)$$Shyo4w2AbmQeK;qSbQh4+p1W&GbOZjW8p3cM{Sfw6WjKV#dn2#X)N5Wmj3SK zx4kzOpH3ax;@uL}emfaY!uQ6)Jt(KUcF2x} zbI8YuD%3NKdUu!?3lFFG4o}9yBPg!J>#^`iayxt!3!g!m9ilNi3Ma3qf1XJVJEq5E zMpI(PoLKlQn%l837CyU{J?B6^s^b`nPZcqlu{0_*YL{{NhH#X3JT*?89h1qWwyDwB zpFs0dx5nZp;y2PF|4*tlj`Ap>Q$|eYT>4qfwjAfK24KLTcD0YSWoCuuGGe`~?)> zC2E&h6xU@@EPggM?@|;C&!O|WY>b8H){?)F{IsUA_<2-6EhQGdh=!y^KABH;TBPS< z$V5I|KuKw(G5Lj*kX9KBUxILC^QF|JtBS=hqPAUIQ6HK{1vH1I)44Q>@~Ds!DT&5Y zOMJGcuK2;<62wh_-7dod@TQ?bT zcndWCuglK1b;gcB&iraG^+G)-;3paR2cQ(=!LiV6u1wtvaXqyCbH#O341)PF`yTW!mX#5AN#{?nSFv33l7ZFGqq?UTB0Ph-Q9Jz1 zs{UzhdQc-i(Oaax;)J5izq+lJU)7bIgK{LZpmG-gec#`oA4tW<(xt$KTm& z#_cTB!FVYgS;j}x;r|KMeyQ>JK-6FgVvSFm95SAoASUu-lh`O|f||Pc$${Q*MAc{- zdYDh5F=zVTc$6>|WrU^AtZp4+PuPy8^`fyi2(4jc6KnN~tV1-IK6vnUf5^Gs@ zXt|L!<6&h1^o5p;!>5_S!X6scHkvEU7@Y>4rr(Z59@7uQ9$(wLW=x(+y=ZW?WFdM| z;$Paylxb>ZdbAlWhuRkPGviqo^X&f{KW?oS$fmwDu)0=*@R@^97Uugt)v-hI|KIsP z5tf@-yK{Bk3GmhQ>M=XLdNiBaJOy+2|9drbF?SAu{kartg?p;9QKba4j@5g6TVEUdzWwG-K4PM2Xe^ zo6%`{V?p(ZH2oy9#q?S;hnfCo@`f{=kq>9dP__Um)95_Jn$=`R^{g9NF%wz~s`DDH zQQe~cm{>h4PKfCZ=l2QdttQ>9VKrVE3(08pG~>Evjxc?CYW1pb#*eABNi~*Fsjk0S z`%J8vYs?sqMq6lMXnVLmm|B@pH4$Z-wY5ez8b@JTICF*GG^@BN-+0%|O=jM$@r|+H ztoCMAF=O7eOSoGJ=Yhx;(^8QoVM?g80C|lc%!+JwBBq1_XsMZFryTF}fj#+s-F4xI_iniaK4C}sLkXmPmCg{!;q?@aWWa2_+>G;683o*0X(@tN_V zNiliM$~O)+ zWKuvEKOB z^og+V)Lz9S&(w?pASaor*PfAjkn`-5rY6sY$I}(9mi>*6caW{><1k z5tvyg@=auoX{VYM#LNq({fwN+b!c1G9>3wv{_x$ZIVG5}Zpy;n4XIw~{+r*!ab;G5 zuuQY^8IMP&uh5_2Tr&ecH#2wCJHnQXX71WEMmUF0gZy;l2xph5hGup#^MScGWAV^E zf9)}GXe)uHo~KQlL^U_o znYqRIIh;jI-NIe#q33fmpBsy7Moic)k?n(^!JOue*UZ{;_^$2m?2h_RwC4zS6Vcf_ z9LqIcIJ{RzE56xlnAOtw%G57x-@jXj{+Iq2)pGz!I&|e5UhTc;e!<_>Gisl@X2!60 znv`h%HG3XYN;Kl9RnL@0!t`phMwt}T^P=@*Ci-Bw4>D_YBw_Z*W|wT_&AJipE23Os ztl7bs(Pl~yr6XGp-I<1atHV8gsLa3hf^e*u@o2`KDJSePX8kww+4Smh8tnj0pA6e7 z+Dq1WB-}0kQ_l$JZnLv9J!2L=P5&@^KC^F~`sZvLwQR3yzZqXl$4prFY!`XPw1jE# zN!7Z;b}p%gy%TevCmOBc{2A^`&3VkUb69s{VYDl%*=L7+C+rE+AsOD+irOt)8xO6uxd&wC zA=3|{zF=B7nz5qQ|KICvdSX~oxc4#F(LOi43l`}#DPhY+7S`l60bNY~%@ws1FO#&tB3!aiWMn0ecjY|0ByX66o@ z@sY8^oRo|O(Hw8QX{61G=kS#$96SFuQ-}M}XtfXbon~~^K4XM-nYNl3TctZx*ZhAk z^S`%QxGI=2^mk{Dzw2qHhlRc9e>q0m|ij!^2Xz4ZLYa9XKV@YP=veUubb+1uQmH@1a4#;BY_{cLJ%X8mwyZN{ouUqg={y0;4F*6Fda(=j${n04#lM(6)B zPQub^`rZ)O^l$s=+NTS%2QurK*(uCMTZLyjGY8bH&Zgx|j|vwM*Su&C6ZHl& z3c_a`W`Azxw3?k>pXxqx=m@EO7K-|RI5tdgjPA3TdvMV@Y;;EVSi&=Bv|b##D+qg| z>Dy5cF*!|bqf~J>{-bo5a+<&=CVM>U6Z0^;EB^}xe|E|Q^`lH@e`;>6# zeW+-@Ful}_;Qv3|_xk%j5%!>*KhH7)sn?(Pw`xB{Jk&>F8=H}2R$DXsPeq%AGegu$ z7a%o!mTY=k*m`DMA3Bd6zSjw7_Gn%4Q3iHur*V^X(9$ z>x}C1&7I90XfnA>KI1uKX)lnC|A!#eJYO7!T;|He4#Q_QxM38=s-?jn~W*$=coteP7f6 zjMW(^#n{^qvNdg9+g{`OzqWy~`|v&xwXV@+TF3Y!tJ(`g;lI9+H=4p2li&0n)4%(~ zJQ(T>d$;jwBpdoLyiZ};Z!lUj^n0l9@LHI9ji~lnjn_jz8w*W8F(pR#Kx*39lxb{^ z>@j`T^rdJd)Of{=GGmD;$+We(G9{RiV0zLZ_|&9DqcAKZYR6&Kc_XbR-=Y4EMoT0Y z<@=`@BJ@#Iw$WWvchftJFGCAWD~I0rcfDg_B!}|u|_s?SGTMOa+x^f*}rkNl9Z)~oezqZZhuBf@&U{hj)sf2*-} zP0ZQXe2c=gz8U!w?M0Xc4~zSYfBRO?Apfmp5BFk zs%-8>)R~I)$fQh#g@gW6502Iu)1zx!@Mqg1Hk=KjdjZw0Z(VNZG>U69w9(DgYwtg` zYBW~NGvC@J)RbJU(Qaa&)ad9&C)cV$*w5-}FVp+}Z~J;}9kuO>^cuU3AOBq2th+F? znR)Dxb?z_y#Ox2vdTj3M{TII+-aBgNuWeUd<2BYNl5^`E8BvcvyeFHTR#eOX#lqV8 zYg-uU{Hy26FuVP?GHYh{S~;wB_L4@AVIB{^176bxHFr%y@6?p>PqOAdv1!E`>BBu_ z&Q4Kn{ud9`&i_vyip?ihIoj{=nWYc z;qaMxiV)q?HGVWJk9iIeemgx}z3XnTx^qVs)%1I_hMAQp`j(XW)`{6Snr9~yQCm~1 znw_`#u5P&7s>_n-iC^^mqUH|D4Cpsc0*$_NP@2(Nh*4Ts4NZ&Bsg@jsabvta;cw0t zVc#&8N3 zo_-6)o8Go#jS?cu~u8Fo%N;lg*Dxp@s}}fMrQaegSvW}Igv!a#}cmG zp`Cy2b!L{W)??MPdRQl06L5yCJ8yqT&H-lb|F3Op##rrdozz^9MJdJ`HQ$nmJP|(I zFuTH<)-?Sn{FY}_19RsnvS;Yul=9c#mN4hgzn=cg_%WwAGiFSmH8reh+o(kIJ~XB$ zhySig&6ujow!g|BZtb7>9Z`q&NIKMG&Q8WJW`u?DY*6=%S67aI?r)L4@QLnpv-a5) z_6zol_Dgk^U!yH2W=Xh5H}_JESL?RR+?5W$hcXr;p>`dkeqUpUnIHeuW`Az)wLh~z zwLe+YV9n8M;@2FtratZ})z&tn`Xpq063)HGSK+rE!?9+5C*|+_6}9NUSOLw^zQ<@b zYr9#s{;s4Zw@K%$#p#b7PIE*+2EH$PRNFIy6^ZBWy18VTNtOX;c_O8BvYQ z%oW`~YJ-xEPr@gq=6*udODAD0{Jq@Yr%kHfyM#}I&36%M?)OCHn{~zPO2VFNMnrVa z%O;0bV`;Df4&HI0?KjRpDJ6E_ryDK!EoQ z)T1~$lIjzF=betG26PNHq+{`Jf#dN`fkt?X!3ik!L_AkGiJDR~d_SxOzMb2WT2X7< z&uoLI{O#~NH|_B|IUOk#zq#2Nzkid4-{$N_-6@?;rXF+(-d|wuX7;JRCw*%5{qocB zJAVDKUmAcuY3`&C#(8BZ`r|OH>m%^Y;S99QnKYWt!kZ7yp)siIIMms!TNA6->vQp) zrpc)P6!^*P@6N9tLFQYK7of+@Mz5Jm7t%bsi00G9cn`xux`ZyJMRXZmPFK)kx{|J< zCA1XpWw?f}rDb#-T~9Yq5#2~P;m>jx({j3nR?tei6>nI$ol58qT19u#YPyT=rZrSb z_t3p`pJmhi^Z-3bYw00cM-NjuJwog0QM`%aG1^Fv(+%0=-Bt(KdP+?}>PoD(N-Lp*Qf>hBxUg+JX0^yiM=WyR?hmqxb0p+D#u;??8kl zEoI?N2$pC0R$v{09YUOSq*dQK%8Iv+wi;N+RPR!bvyQhCtVUL2>jW#&I?-xkon$q& znpw@Q7FLqg(rRV3wvw$j)w`J#tG(61>S(1}ovhAQ7c0%`YIU=^Tj|(&^sr8`dRo1# z-c}zg!|H3DYW1@+t<$X2t^QV)H309E7-VHzgRLRfP%Fn8W(~JSSR<`7tWnmP)@bW2 z>ul>BYm7D48fT5Sa;*v0L~D|jXPs-EXHB;9ttr-2YnoMHoo{wiR-rZ1y1<%c&9>%P zbFB-ldDcbNeCuLsfwj=O#Jbd4WL;)mZe3w5wyw0UvX)p&t*fnTtZS`h)^*nP)(uvX zb)$8Yb+c7$Ew^s5R#+>oTdmuy+pQAo4r`Tlr?uL;%evcIW0hL>Sod1@S!LG!)&tgq z)>`W!Yn}D5Rc<|Et+yVvHdv2Y8?DE!P1Y0Elh#wwsl(c8J!?H@J#ST5FIX>HFIn5H zm#tT-TYIcetWT}a zti9If*gt-0?X$kJzP7%x_FLat-&x;V2dp2gAFZFPD(h$K7wcE+p!J*eyY&ZNs$to- z?byubwy>qGY;C)?XZvceT6O z-R*SyWV?rbirv%hW%s7v>^}5|oniO2Pqq8mnf7V+>2`lR%N}44v8yR7uqxJ3+!3;Y`U!M_GR|v_7(PG`og}_ zzRF%=FSW0>ud%PSm)Y0Z*W>Q<=XQ~OqkWTovt4X2w{Njm*emT@?c40z?GpPAdzF2s zz1qIZzS~}7m)iH(_uBW_W%m8{1NMXVTKge;o&B&~Za-qLw;#1P*pJy8akkrJKVd&< zKV@&WpSHKy&)8e-XYJ?g=c5(h-e$jSzhb{?SK6=HuiJ0f+wC{)x9lDEPWx^99s6B- zm;IjozWsr{+y2o0$o|;gV}F9R`7^B1pW9#9U)uZZuk5ewZ|wc{xAu4T_x1t%2m43+ zC%el2+5W};)jnwdX8&&g;gDlFw&OU=;f`>mqa5wHj_3GJ;2h!9bK;yMo%+sEPP}up z)4(~#Y3Lm59OoSGBsh(n#?A>&qI06t#5u`n>NInjJ1v|fr=`=%Y3(FCZJf4FJ150y z?{siFI;l=4r?b<=Npre7-JI@Dx^uGA!#Tz2>GX1XJAIrCr>}FW)6dCtPIFFo`a4<9 z0B4{x$jPRkoWafzXQ-3o40DD%Bb<@W8O|u@OlP!nmUFgqjx)v?OFui~obgVsGr^hY zOmgy^bDi^?$xgmA#hL0%a|)dEo$1aDr_h<{T;R-dW;=77xz2^oJm(^3zH_m&z**>A z;#`V1dRIA@IhQ+EIE$StovWNB&Qj-U=Nji)XPI-IbG>teQ{>#}+~nNs6g$hETbvcn zO6OMRHs^My#JR&+<=pA4cJ6ZScGfth&OOe(&V5dqbHDR|^PscVdB|DkJnWP^k2vd{ zN1YAMW6nnBac7hBg!82Hl(X4++S%ef<7{=Fb)IvccPgA0oEM#!oNdm_&MVHVPNnmj z^Sbkfv)y^qdCS@1>~!9C-f`Y_b~*1k?>iqjyPXf6kDQO4JrWt}JFVhcU*PV3H}O znae!pvw$7J>ajR>B&*MkV)5)~)_@(u8nR>AaqM`Oz#6f}>;#s`PGn8kNvtVr#+tJh zEQz&btypW8j1yB^){dpH_N)Wz$WmD+)|qu-X{;;j#=5g~b~5Y1PGLP+FV>s&VHvD1 zJC*fgnd~%nI_r=3R{p{Uuz_q4%VvYw5H^(MuwiUC8^K1hGuSA0CL7JpVrR2+*cdjJ zjbr0kE}Ot6vPmqDoy*Q+lUY8S!ltrmtbm=*rn4EWkj=!Oo14XEvpH-oyO7Oe7qR*5 zVzz)SWS6i@*&=osyPRFY7PBkaRcr}c%C2VDuxr^eb{)H(-N1_2jqE0NGb?7x*)41Z zTgh%^x3Sw<3A=->Vt2CD>@IdUTf<7(J?vg~A1h<`vj^CNY%P0;tz!?fa`p&Y&mLtP z*kf!Xdz@`zPp~K1Q*1MPnr&gvu&wM__8fbjRj?P>i|i$~jlIlXVXv}E_8NPgy}`D# zH`!Zk2iwWsX78|f*)H}Td!K#4cC!!JN9<#^hke36WuLLV>~r=7`;zTrU$L**H*7!q zmVL**r;phI_5=Ho{lu!+&+HfWD?5lkb@n^^gA=#7%^l7-=YmVFxaKbRxX%NA1h2>A z_>sImKZ?imqj>{<3~$Jf<;U^kc>-_58}k!*B0rHg;V02YyeV(SoAVYtiMQmfcx#@_ z+wiu$9Z%uyc?aH+r}9p`Gw;IFcvs$ycjxK+WZr|H!h7;wyf^Q|Gk9NqD(}ZL`Dy%g z-k)dj0em1I#IyNeK7UBxp35ijiF^{z z{Gb_;vhxegiM!H}aeK&AgZ|=eO_`d?mk?-^Op}CHxM) zir>js^Sk)nd<`$<_wal9eY}j{&mZ6q^0oXSzK%c4%lRXGJ%5yM;E(Z*{BgdCKf#~m zPw~zCX}*O&!?*Hh`E&evUcq1BFY=f8HvTeyg}=%x`D^@j{s!O9-{f!c9egK$o4>=~ z<-7QM{C)lb-_1YdAMua*9{vgclz+ze^3VAf{7b%%f5pG%-|+qXTmBvYo*&>p@E`e4 zyo&$Kf8oFKgZwxCJO4wFu!Jof!2}mVNTGxluJD8}0&#?>C*s7BqP{pv#EYXv196OK zD2^4!iQ`3rXe1ho6GWmoQ8W=JiKe2NXf9faB+*i|60JqDXhUC%wxXR#5$#0>(NUy| zPNK8uBGN=x(M@z0>EdM3L!2Udie93(=p!;jUvaAFCo;up;&jnpWQhS{pco{w#bEkM z3=u;`ju<9}ixFa^I75sQXNu9{EOE9tM~o3;#W*oup(*pC}XeiwDGmVy$>ctP>B5a`A{*FCG;e#A9NkcwB4}PtYggN%54}ES?ry=%9E; zY!%On=fv}(LcAbe6fcQw;$`uQcvV!2*Tn1M4Y6IkDc%x0#7^KQ;zRL~_*m=_pNLPzXJW7TTznzE6#K+i;%o7Z*e|{n--++V0r7+QQT!yT#Lwav z@vAr}eiOfoKO{*@+R~9saw(*gN^0p!Px>;DN630IP97=i%cEqxJX$u8$H<2ASb3a0 zUM9#!vavitCdw0K6M2$sDx1mXvV}~NEoCd&S|-ajvaM_vXksAyT~-z zRd$ozWx70B_K>H@p0bzhE&Ip}*;k$_`^ijsnmk?hmsxUv94H6LY&lpCkwayU943d$ z5ptwFLynSX%F*&HdA2-9j*(;KI5}SC$_a9!oFwz)x$-TyBz2$S37fa4OD|vwi>L4sG%xH4O7F_2sKijp+?~q z!K2k#>TGq68bhC|v1*(euX5D{HBn7cdFotso|>%k)f6>VO;ZKxd^KIoP=#uyxa=->MFHFEmc>mYt*%BnYs?I&%Hqv zsTSk4}maALT3bj(*s%}%as}gmGTBYt(tJPift-2ept}Rvf;03k!(RZp$-LD={ z53055A+=6DOy8?=^$1=t`>5KW9#b3D<7$(7LOrRTQk&J&YKwYCZB@^z=hX9f@#_nC z+3QPcn|fKjqFz;%>NUL3^$oRMy{X<(JJe3~wt7dst9GgP)cfiKwOf6tK2jg6J?azn zsrpRqRiCRb)R$_X`bvFG2h=z8gW9jYRo|)a@dDN#)Q|L|`bkx(pVcqwS9MVRrhZp{ zXuNDy+uG4gb1k&gN^9+EPy0I1N9cMwP9Lf3>!WnMK3X@>$LNOoSbdy6UMJ{Ay0Jb% zC+ZV*6Md3ys+;NNx`j^CEp;p1S|{r^x~*=fQ*?XXL3h-tx|8m#yXZ9CRd>_fb-F%T z_t2;4p1POrt^4Q<-B+Kg`{_)5nm%3k*I9ai9;gTDY&}>H(L;5P9;S!u5qhLPLyyvD z>e2cveYQSFkI`fGI6YqH>Ir(Ho}}~ixwKcGrzh)tJw;E|({zD8Ur*OFbfKQ9FVM5} zY(0m*(R1~MdY-;W&(|001$v>rL|>{G>C5!x`U<^RU#YLsOY~BGwZ2AQtC#8P^!54% zU8HZ+H|d*ov0kok(JS;yeXG7r->ysa9eS0%Q?J%{>AUqBU8?WV_v-s}nZ92?pdZw0 z^+S4{epr|5NA!CAsNSF-(;M~UdXs)aKdGP6oAuLri+)CL)z9kZ^z*txzo1{#FX?Ui zW&MhNRaffQ^y~T!yG$;e`UAaNf2cpwAL~8(6aA_FOz+j7 z>o4?|dY}GEf33gK`}Mc_JN>;rpnuRm>YsF#{#pN`f7J)|Z~Axr2YzkOa&6afnaf?_ zN>{nsbzRT(-M~G?zMN4oXhquhA+Xt#lTjN6d*yT`i6xyQQ+ZX>s`dxD$jp6E7l zPjZ{O&D`d03pdGa>9%rPyUA`Fx2@aGO>x`19o&v?s@uu!>~?X}+^%jnx4WC}p6vE; zPjP#?z1-ezA2-A8>z?ZNb2Hu3+|%9uZk9X19q0~nv)#e&5O=7X;|_C&yCd9@?iubV z_e^)RdzO2)dyYHC9qW#B$Gf@i1b3o4$<1@mb6}#?#1o`ccFWUd#Ss~z0AGby~17WUg=)tE^(K-SG(7^*SgEx z>)h+z8{8uIM)xN7X1CZ~?%v|Aa96swy0^KvyCv=&?ke|AceQ($d$+sBEp_j4?{)8U z%iR0j2iynUweCaiI`?6>+_2oa36Cwx{te?+$Y>8-KX5m?$hoT_ZfGq`>gw% z`@CD>zTm#-zT|FmUv^({Uv(?p*WB0LH{9*+o99zuDi>9&wbzhz}@YB z=zipW?CxuXn1~&&%{q^G^5rds*H9Z=g5G%k~C) zL%gA0jyKF3?v3z9dS`f}yfeMg-dWz+-Z|bFZ>%@Y8}H?M6TFGuBrnf9*E`Rf?B#n? zys6$aufRLso9@l<3cZ=$1>P)gwl~L{>s{#0^Dgq{dl!2PyoKH+-lg6m?=tUl?+S0R zccpigx5QiOUF}`tUF$9LuJf+fPqu?v;3Vc&ofS zz17}b-re3BuhhH8yVtwVEA#I69`GLY)_M%51(a_Fx8r^1k-I@%DS)df$29 zdk4H9ydS-vyejW!?-%b^@1Xaa_q+FpPrl{bzT-2W`@)yL^0n{!p6~mCe}rGpkMoc8 z>-$If@&3_%1OFJmp?|D@oPWHZ;5YIc`zQE`{)v7Q|0KVu-^_3BxA2qvmVPV0wV&*_ z@!R_C{1m^v-@)(br}~}z&VCm^&F|`W^Sk@${>gq1{}jKc-^=gq_wh6QzW%9xKR?qy z%|G4m?`Qb~{DJ-;KiePd5AlckIsPzzxIe-l>7U_`^3U`~`)B!Q`{(#${IUKxf4raT zPw*%Dll(mYT>m_OvY+oy@u&LJ`~v@cf4V=zFZ5^n7x=UM+5Q}Vu79CF&%el@?_caM z@E7`*_?P;N{LB2y{VV*%{+0e!{t|zwf3<&&f33gFzs|qjzrio^Z}e~SZ}yA*<^C=H z3V)@4tACq+yI_6>q@t^Ux`p^2$`Oo_m{tNz#{!9Kg|7HIb|5d-z zf6ag0f5YGIzv;i_@9=l}Z~O1~@A|v^_x$($5B%N!hyF+Y$NnDw6aQ2HGk>rDx&MX# zrN7Vr%KzH`#^3LM>wo8e?;r4g@PG7w@~ixx{a^fF{e%8*{_p-D0R>iI2Ts5O9*95& zD$s!&c!3`T!4W~dATBsEs2>~^#0N(Q4T58WhQYDHal!FHLeMB^9Gnm&1}6qhf|G)# zL9?KF&>~0*S_Z9x)VN? zL9d{9&?m?U`Ua;4{esNkwBYohe~=Xn2nGg&g6v>$FeDfnDmXJ3 z9h?=M9h?)43C0HFg7HCaFd>*2ObYUXbA$7O$w7WFC72pa3krhsgXzJHpfH#jToB9( zW(RYExxt0Oyx^i>esFQHAXpe&5?mTA3N8yS53UFn2UiAH1xtdZ!PUVv!L`A%;JV=Y z;D(?mxG}gXxH%{emIt>4D}t55t-)=gww5nlI^nY46K=U*3DE_Z7Xb?0r@5t9xJ5``X^u z^}fFM4ZUydeN*q7d*9Og*50@EzPU!UL8I#d{Fq{@S5-;;X}iR zg%1xO5ndZUGJI6{=&%(&CZr(?c_=~|`e6`;VHC#UW5dUVj}M;^J~4b!cqmN5G|WO3 z>M#!vheyJr;dSBl;c|FGcw=}hyeYgnJRYuuC&H8AE#ax~*6_CQ_VCH!Q^Kc)PYX|n zPY>JSYS;<8VG*u{>)}S&3-1Weg#B18$g})sBO8D&XIpME{cZa_g{(ATu;cte|4WAeOR`~qzx5M8FUl9Io z_WGnfYBu%m;Pl}{W`pF;}CZlAWd~EV@$;T(3kbGkDNy$UWB$+0& zq)O^!o;;j9l02HcE_r=&IeA0!#^kZ&P05>+$CE3`6UmdwTau@ewN9mzAvesVK8NDh-*$vcyyrm?hbld6OAoufQt2gFTY9R} zQ-G9n+AZByx&&6+W0fAa*~Z<{s&NUXw$nT<*0!%6EuLMlvguHj&Q2J ztSg4y`iV+e#t`2O`N_1fyb-l--V58!Y)H0^6rJ?-qwQxGO*&35a7oqx9@yU5JK8zD zd2N63t_En65Oe7)px*$RDdp1`Bdb!wvzb~zn`wGBQwwM_jn8Ik0m>8^zR8r3^P4pB znYPUf!FiKIy=VbV+u(Wc88dYNZKkvI0RMZbHdBkBYH9)A3-EhWFcGhLMVB-Ynhf`C z@9r&*7RP(X4JdXj#pGB9V3sTcFkKozuzAxY?{BA310%V{lQV!xi+{`!XV}>sZ8NN+ z!*7>-(s}=0aTd@{$5_~S-ScWv^dC4p+P${8xp%O4(6aV@V*QQ;x^J7@-vPxjunEwC zy}UaII&iT^)tzf9za+ly6XV@h6ufsEY_cSG#S%+mM6jhPF4$7DhDv0eZX3=m9oT~{ z!*rl54W2m*i)?ci7CEN_yO+$}k^ivTBqjQSxM?bZfeO zG(CKdnlaKg?A1V{*Dr{DKbWBE7h6vp-a?6(Yn~Iy$#5+FXJoabR%SSsY!x$WLuta2u<#Gz2DtG1ZcCq_CV=T~vnfnRu21wOCI1d0+vlkd|}P zbqu}%RWvPYR&2-1p8#$y!56eKBYO{F@ukwZTWp5`S2$twb#bz-(Icu= z--w1bIO_G7JuFLWE6;|Nk*ROstQyZOSx~Dj2BV8Rtu}UHXZP@AduL~HaAJ+rsfaoe z$zxE>Gc`H3ZhI?WGf@lz&3l=*u}oK zX5`4uhw;!B`D#2|Dv@U#2h22$7l%D9`x=0f#3+eaSyv`q%d*;f!{Nd8!~H8K&+jkD zoVn;Xs8(lN$EUZ5Qd(Y|b?7xaOD|PC(Ttxtg*dQz!GX<2!QL-3}H+Ofov3c5>#lJ?ZI@LZX<1DH<@R(i3KUm@rl-S@${r*Q&Ey(!bE9`Qznvi;;a9KTpOefKEUkZVgEy7b0HLIA;`y^jkv89xQ1hR6yEh(JJ z%BPDaNS~`y-jAyhrVqtN-d8pj0~T5Z`eeXr?EFmGn{%N8=UM;Is^%N5Czz}?t0&TZ zUn`3^!@&`;ajfSYl~Puja?CFEsr!KcM0e6Zs|szRn?hP)mf{m z&ZMf&T2*x>Rdv>?sxBzf!CKwr7$mzKGh45hIEL$ZooR_mvkUcEj`68Z%GH_DQ|AY{q^tH_6)jjQ02Sl+O>U@yoQp()2T8@#|4d3%3A4X^lTPRe?Df7-@If;2!=dRR15#WZP&cp>2x1gOo?X*hi#3jccEpfdpDV2bS(sOccRaS9Zg86T@KHru=NQsZ; z{fpOI6>IH!XYSQOM<-WKj?7$-?SydmMPBRgfSjIBjp&mhTbxX}B7&$?R=GXG_Md_*$eO5IP1|`)zV>ax$wE~u0>Iamb zT(a%tCWB5+Xpoi@ zA)#w~c2u^>KsooQWAzjgn(Vdb7!8JHpR%#erEug@I4*=;?-@OIO2e4DgmqUxD{yi^{-Y}DkBy#!vK4z|~wcrtPq#!Az-tnc1? zM+AY*jJ4qi_+zW!BIY7)F3q-@j5glU9Z=ZBhvKUxK8THYKKqWwJZ&xH{W;;~xC=-B zO-F#Ssw%NB zx6M#8IR$G@9-KxMo6nrl@NmvxYtCdyi!(x7`w-fE1}9UG$+%a?TFJ4<$8jD*^+TQ4 zj~0uA{q2L@y&WT7SO^n(PqN@-Or3+fqWECesO zz|h($^GNA4k~RhmkgVMlAhj+8Wwp^3w6Ja?%r{?uh9>B{6+-B=(ST}Q&&aKyg%KD7 zZrjT~jd+knvfyy153%DNC+SKl)1@2xed9SJsr%Z=m6>-{gyQ&uV1sq-d?UFba;<*w zgY6w-*e+RVUT16jw#aWX;J3|v?s8_W!MD(g7j?jP#VfSp6~5xd1)(misTeNh)rwkZ zMMWd+CU0dQ3*8!}#xjSpykk_uZwo8w?rhAC%I{9qEJ;1C08in5y0D69PlZ^X`2JiEmdla!*?k>=D!z!IEl zIrE`@je)Dhy6#ssvPB1KvXS5#88{PGFV3jUtDy)(@1-2_@CE2ex|Bv8P)W-_ttC(s zM6J1~HJ7NH*&#+hGr1x8kY|m-Tzoa0uN^c43@iS4?ZC#%W=~B)4IH!+))uQShnokT z@wUtrE%wmDB-tAJamSI9;X%Ke>-#YG-RV7xgX@>@5B7@V#Xj!O#CvZSBcQW=1);O` z2z&kE%{bokjC!sNw>gIi)kp2Fnrn}nv%}b|b6#ZYB7!Y>b_4pCwv~)IOA2aU$AvO#FFsf*x zgP@Yf5}*nBzN|UEWv$hcjEf~g=j_Ix7(FI+1Y8UTy~7))xFvaXdUJpK^n^G$^H(~Q zsoF!CN+&Q?djeDGN^BLc#2ORUm5y$z_UML(8PMP7Ylj{Ey`r?KOPe=1<D^z*2(k7 zi<7M@5%%`Q(cyUqMXXVc^#V$}4K~bylhwl;herpZHXEs_AZ2xJjb}9vUyv?VkN2+A z9t>qn==0zP=gvl1>_Qw)M#`uTqHRjDJg0#Z;nrif76&+ewAL#frB!}=fD8*x6GKwR z!4fA9Oy|AF?!a>VT30&NfzhC0CLQ5a?Ga9;n^0AI6ROhLPt~6N=;R|$>$S(aEGP2# z!Itj@%XC6*IwYUL_x&cs}|KytsM;%0d+DcDn5SHNFA8Xkt=D{~>9V{|leXPSGRst*`GS^zpc-R9Lk<^$- z4;k{*+_waNB?}~qvnS8Z{2s$)UMO^C0L5~dN6g?4Uh8+^*zFn5;XAmjwhd?4#H~*_(3C9I1|Zjp z-p?khZ}!s2WdjVA(?kwx(c)`|%0R&|1Iex9StSE`&U<>;FV5DE+GQUdCbMS6+EHiu zCu$(0RtTMs*Zpeq=*;qujf?t00Sp#0S^`5YKXz*i@_Zy^s@l3tQ-7q{Tvpti+cxya zsDMxm${Rk}go_kZYw?>KfOra&=2Plev`}ESILnc!`7~}cuvy^bX;9I3XMmfvZ9`90`Q=-KzS4cy$7FJ%fdDc}V}d)N#|?$DdeFakylg}Q%yNmb}J+lj&^jO1HP=#Vfb<*654gYUR;3vvy`CJ<%J@0YCX9qJVS!W;$BT z$mSSV;{${eQxuLV_hXP-oT$a1+B`AV>|T76C8_p3srHt&HiNZnlzBE?V_~uu2P(~Y zg90MIzjA_;C`xyoD9v1aPAXp5!jqZDPvTNTLTWaSG-tJ2vXRBs+-LG*xc# z*hGso2W6^bOWCCI2~RO6G)820U_0j;fFn<9AG3%pOtc-&M@GfDZ!$_*pi|mY%e=Lr zmN?aEEYR((eLm29&ybJR8}xk}80; zNNbX8`^5m}vfH&=*_L4ySv&1&qLAx6yojnq9Yu6k0oCmv%y@P|RzYm8l^(2~#=G;z zcr|xKFi~n<(^Z0+42jygYSwYntRoMF%StF@fHS9s;BP0@c=fbZDc$DGofT9_wOiZb z273_2TBXTV`9ZMO-%7U6+;4qO$Mz(on!;Xpa}VYbUQXC2JY>2)?<}mKM4j zmAhEf!^VnB7fxN&3mKX!T71x}QSZq+jFgQhgzsSO$?ixMYfpAZYPA03(n$G;P}-n! zjSwjbF|_(*XQZ5-#%dTDOCuyN)iTl4*wR0#ky5xkQdl&_eH4oG`04`m0uPKq=gid2X+}09=Mqq?CJWz3+!AXE>I-&X& zb%-OocZ9Q1m~}Hx+Gyr>(;VKxmcmg}(*_BQ zU>GB5?{4R%6$Vn>m=#K}aGDNR4{t86Z;PjwJ{ z>I{OKmAd*obJd>;4CWYVms0J*X4RR^5Q8RIV5P@pK=q?$l4fqq9Z5@dt9B{q7f;-N zOnY)#>Mc@6RRFAPLxGc$;+~ymV`)+dYZ99*d8-GbY#4#$n(w|k^9UO^0S%5ev{B?T zjB;#asQp!0vJ1K#M0pCunX6MdFV>zXQJ-&#YNJy620(9ra@}4lxnuwVE)}&l>NOW2 z2qy8!ynp`rrzr{V^T=(Ya85l7&(x@AS<{*H+S=)9c4=HnF!H0zwBfm;r=z2ZS?lMi z^s9KRcJu14vDjqjvfU3i%gAyVXN|9M0qR%CZ3i0S%23KU)b~dmuDy(+2AJYNdF^c^ z;@g&q2x5@VIq&ykG$nm`l?mU;^SNv^rS95g(?WzPI)p)XD-Npz1yre*1EuRfTqMvD z&bJ|z4s>74;Jt?lJtUF}Ir93+H2*h(TVhqndrmjB<*7s7aE+6Db{9ll@leOrLdxl278U2Xj#qO4oI=sg89y?qJbjMC|a;DF{H4$;n=0*E~Kq|o)XDC9iPDgrZ@KtgDAMJOGZ1{^fd-?+91KNm({i?M+S>om~hiVehXZNDk;GPjqG2Ir6W z-bH%Hh|w-aI<=v{-Y_+$?+A!CFb&BH@D4zW?f6$d`tK)7{v`*>#`VgWWV+ErbaxeTLS zaWc&6jm5impfK!SwTC&dG)ymIp2jdr#n=+!Gr?vN7SLO)47ggg*xp?n1$4EOj3v_C zZ#0j_(H4Um##>MA?JoFnYE3{FYns3d?>sgU!`MXcou_CZy8k&kUKrVtyRsYz9WmrC zRm!$Ps@;%9enU3G;3kF&R4&n+S2PT}HMOFo9pxL_2WWSHvA4}~R2%S^(}o(|uzGS_ z>E?IpzmRt}1b~wzQm)B>Q+f`07?%%e^~ej}+QacLCsx6mc1W|HLN65xS{;y|!mMCF z&<=H9y*faeJEdA{d_ipw=N77;SJ;pyK80@i6g=>NCjwhj5VGEaCuvBt-h$UMkR}@! zGG`VtR~K46bV7!F!k612q(vW{jnxgy>e4k{#W~AjC6ih)Go1>2opnEQ(amN#uWRF_ zT&JXlMQ28?^2mJy&g(eEhb$b^EX={wWf|Sq!pSkWNU+)1_gYt?@QTIKvUKOLHY-hH zIGnC_9E4Z5w7)JxVCrw z^k}h*<9Ms8<`t8?7V4>6^eUp1a{mVYw64LQfmMk&Wkj{m9jys9JeUL6WUYtV@@xRO z_>Ql&kS40~cz`t9tHtkn`42c3WxeG_ntZA^+_)OZQOLUlMKV1lg;2IZ24#)&Avu&z^9M}MrTNoMPjGjj0XEHbfo98#H~{!QUi=2E^V?Ss zpGCLrL&Oh8evKsc3G~P=#l&#y$tR;96h<#M(aV=(LG+7Uja#8KY|dv}SNEUWKYVWM z6n}1m-Q3%a8#Syjo-Yb|OUC?d0vQc6WQ?8k0ym%mSk1QsQ6d_+x;ZK32iX>?w#x_w z<6HZ&p5`k`HeV6vS9WheVPfTC89O3~shBVenI~9^^uc$hT8xTnh>~janiB`13DC({ zw=4<|(}6~IaHruOJbj1Z9z1=A;T}AFr{Ts}rnUhp(!tj3`71D@p1k4cnaTdiD?q^X zHYhHGJ#xa)XlBCP+%gn>AI4_5!4fSD)U8qVSvl@pdKs6HL<-Q3&f$sj}{x19w~x-l6|kU?@%c`F8KJ35TFw1UAf zZAXXkPOtIR7H_M=X=P=cR!yQUg*j#t3ZXF5%@Y_`#c^89sF!2xnJ?F?{(?#eM5IJE z_w`s5xCj*PXhxUtib5hIdKZc_%Ua3xcCJWE_94aVjQCa4Xns~|0*@93p3yXNn5NOZ zJB^S)Ms;dV0m4cmeHy9hH4}&unnqT21affubvsaucLxd(l+-k`cuR6LP_vVs%za#Xr=F4d7%IU2b$6@ot3FYyYPzDj9 zJl=LWLL#A9&Ix6yn25_ulW0qU6K$pk8FMx{+U&B8_3WuyzpIv_x3+ zmewfpub%%FI-@V#wU56^k#%1?8(?LV8JekKkI89z6GBg$z{A2vtnx~|<<1G^B`{^t zE_6|2Dx+KB*Hm~84B0`Uk?Z)QWF5#i!dx zp-Eg>T+g!QWWJm%mXqajvcHtfm#fN`tIC(F%9pFkm#Zq4t16bODweA%ma8h3t16eP zDwnG&m#Zq5t16eP>MvK-U#_aZTvdO$s(v31%ETHNm24@MFQtm5RJoLrU%`63b##2P zeeIg9t9dvAl^DO{{EU zeZ@viiP4sq7;Slp(UzAOZFz}dQv!ZYRm-p`F>FcXUv*4Fcs&Zv;+e`9NqZB0}jU z{$hmpu_NxR2Y04U;c-p}FN0!y9#a>tHe_BKXle7;EPd^m`?}hRdygC2M@NUx?H)d7 zFI#YOV3?x?Zr0%CeBE-K-cpO&pp-RBp5EF$*i??)gR932g7>jd#saYX+2W6od#`*x zhnILXc@$SoWuZknm3zek4j9guZLpVWVNhYY0cl_|gyfwK2)?x&kbw+VED+V%&IY6l zMAh7Yv_SUfDo#Qove-LXKVVHa7C>kO$+JmKGGK2IsmV?vLBiqSwcmV?#OaalSNDw$ z%_naM^-_uS_t5f+0JyOb%!{TZA~aUOBTZAC;jYP82cfYN;HDKJZLEZgwZfY_PNaoD zwI{qUp^-1w1Nxi?&v^C4e&fKy2F<cX1^9e=8m3pUA@$Ly6w6U{#vI-kInQ_2- z1GrB{q6~N&B_Hq{4QCie1U%R!F$?Y^kS37}Y4`;-b-*rum$2fDw`zadT00lCa@8;=9v@lO~nz9|k7o_Y-T`_EYv7^yB1$ zB}()rpQLUL(3wE0GcWw&O3a7$*#C^?s(HraO@yR(R!|$99=;1&l4JLl=WAINr`_A>`A zO-+;;boJVB4KmU$A_S=8hU;(^b(j3dI}doY**>cj?lwgl#0v0?q^-8-KpjbBh~uHW z-Y%d%_Ha%b`NznEA4WkGzc}Q-5P_8c4nzl}pUg3*efFb6B(}iquVgHNaWPpxQM3fc z#q~RZaV7Z<;N@Qp0dmdnW(Y(_6kqtDo^qUrE@?sEi ziQ8K`-ihUH8r=@-(5A`PAxs17P{oG^zs$nVcn;p_SnPq_xxXEQ&QGw0Z!1ebn`C>Z%rdXs@AI}(w)v%v26lE*V;<%}W)(lGp&>8f z2>?<5{R~0j1q9HQtI8Fh`nFM@^?=5*cB0t%jRa+cN9@j&_u_!I@&itCyuDGAA^!4# z<5t3PJcR~++MtP+F79v#*2NB7|F<=iE`i{nB6inIgn9F`45|o}kEpEdv=zseH+Jl> z8nMuE>~9}<&lR$(_3{rm{Qoa@_xAUFtpr(GtR{{MoKQ3f!p}d-nJ|R^ZjVOy{TyqI zQtQcIpCL?})dAl4y-@VL?g(TH8rbwRIB=#KXR zu7tf0fI2#eIu6KSBvdt^{IVZAfjQ>Ui8+N=pY?YYAWUUXG|Yq%P>KGvJE9sMD5hDJ3lGvMu)k7DjoWFR84!9&5g&F6Yks&wpd z3*T7?QD-cSzgLS+ccP90ezc}325;(3y*Q~CVsL`)(Xwb;YZWc{Klco9f!;=qb*?Fm zPDPwHuI(SS0-Rt)MkA7QrB9z-8zZ=!P$zT(|(F|n|tXN zWFwd3b^v>imkZ8zxQyHId*1}4N*Rg(Uar?OLU= zgb}3U))p0B+;zvg5%W&9t9$RjXH5&o4SGx5Ig`e-W_;jr7SNSgV6*X|+T`oU5z|;S zo=yP@9oMX$8D3dEbBw~c621O5U*SSbRdt-G>MJhMJF&n~9hPg*EZfxzK3Tj{I7Drt zCOOr+hUMVi)HA?Ysz>4c6w#Gul!iXu$an1bTd0>shS+4%buOx0@I2&Y5%mIyI*t&) z9)-I+uN?EdnR)Ous>;y!1`2PY4YaW^L|>u})Cg)F?l-avbGPXw{E#+zi-sa4E;>6G zvu*09JT1X5zv*J&8J3o!3&<3MYI_& z819u9__Ty?HyEWEqgJBUvO!OVo*&K(4d@iyz)zOMAIx>yHLQa696>}|v`u|W4sNF% z7|PiUZXx4fQZ%$44FOtPz_}jwJ2B`w-~%i~n4I|`c_Aj@hA&1rvC69j%v{5{v0YwJfJiKjRQ9IYe8CgSDcmq@&I8$`a#fsxlM7`Q+RFj3~(XnQFsDGbc0?15eprEfXahP980KgQB0gS zr+@kkY`qOcJty8uwjFLZAb>V=84dtZ!|I_=LA)aMZ)nj;1ad|wXB47x=ccx!r#3`a zZh0E6+7!i5c_$I|(Sa)J7{ZHzV~DvB+dF>}eN}i>aL=YF7Q;@|t3vq}+Hgp8&eqh> z?(NDl5`X2L{t52Nn#vpGP-8QO@KhEH9O2If4lM8-!i&OnR2#LyqgOEj(OItqQ70p! z-XNlp1WDJ)SW{tVq)iJnp^VAVNTv@3_9K3lmh(_VF5?9Kk>-b4ofrb+l-ZeUSwgcBH zVmOvYu?JUpu$9@VmI@?pc{)W8Tj6H{4*+Td8cl1vDJfU=dsAs^ozo`*^P-2bGw>8G}#`8W4OoYHii2UWg!zp2^@P<5aKz;1^+-S12L)e%D z;WNOuTbT4tSg01B7oxL4DsNN>@NGlx=%zx<3f~ln+H1WSU2=Hgc`#^AwCNe7tdj{* z?>eGQUR;Rss&Dg)q56Q7R|1C;4N~w5pbYVGf19_dIzAr9oUzfu9K==fCQ~~lGd$N- zFXQ9fChE=D!&*Xtwo?z$ShrpFkk7z5e(%N786P6gGd@I~>qkcs^70H^53rmr(a6;Z zzVlKI{8z9ZzH^`Jn_kGTcJDZ*@(mw!Bh5=q^6!q{YZpGe$%gWzd%?F@ z_|D5w*x?&p3gMOW`3Jre&zzUCfVYnqFFWCiBgc;q$&sJ^hVNq1nXbO(u=X;|%Tb`4 z`p=i$s8{snt2MZobG+3)oxvfWkGJ|wPXHK4)kh~QD6cdhvLnrYsP;d39Uex&m!q=}cDA%d?s9{N=@q}5;gSw>ppNrwO^ulY%TcW6(1&zrkQ zi~s2~WsV!VY@e4qkrqC528y)G(=Qj&Do_7hNUPuU@Q$?jj+aQ0);#1TQlv!}I<^2l zoL4w@7apW54k?+Y$9VKh<@lQ%NDJ?JaPIZ*J@@sGzxBnt?)tp9z2~kkI)Co`xt&`t zo?D+jwDAR>`OH(Z?A-df*IN40=bS$}|7qu6cYd_~+SOOBtX~lL^VdFO?Z(;@Ym?RN zp4Z%M;4qzReBev(_tMK>y6>g%(1n-QUqUSxk^ez_IQP)f|M9~YoxA7Ux$4E&-1Y7^ zzx6#2z5A^(y<5Hb;W7Q*^XCXSSH10(FMe1AeewCvI(OH~L+{>s!gAt6{O_Sl?>>KW r{?pH2KL4@nAGP}GmG$u2_ulnqpZ%Vd&wcUkbJZ8F?yjAys&oGzvQ3ZV literal 0 HcmV?d00001 diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index e954ada..20954a0 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -6,6 +6,13 @@ document layout analysis (segmentation) with output in PAGE-XML """ +# cannot use importlib.resources until we move to 3.9+ forimportlib.resources.files +import sys +if sys.version_info < (3, 10): + import importlib_resources +else: + import importlib.resources as importlib_resources + from difflib import SequenceMatcher as sq from PIL import Image, ImageDraw, ImageFont import math @@ -5638,8 +5645,10 @@ class Eynollah_ocr: if dir_out_image_text: - font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! - font = ImageFont.truetype(font_path, 40) + #font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! + font = importlib_resources.files(__package__) / "Charis-Regular.ttf" + with importlib_resources.as_file(font) as font: + font = ImageFont.truetype(font=font, size=40) for indexer_text, bb_ind in enumerate(total_bb_coordinates): @@ -5649,7 +5658,7 @@ class Eynollah_ocr: w_bb = bb_ind[2] h_bb = bb_ind[3] - font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) + font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font.path, w_bb, int(h_bb*0.4) ) ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) @@ -6135,8 +6144,10 @@ class Eynollah_ocr: if dir_out_image_text: - font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! - font = ImageFont.truetype(font_path, 40) + #font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! + font = importlib_resources.files(__package__) / "Charis-Regular.ttf" + with importlib_resources.as_file(font) as font: + font = ImageFont.truetype(font=font, size=40) for indexer_text, bb_ind in enumerate(total_bb_coordinates): @@ -6146,7 +6157,7 @@ class Eynollah_ocr: w_bb = bb_ind[2] h_bb = bb_ind[3] - font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) + font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font.path, w_bb, int(h_bb*0.4) ) ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) diff --git a/tests/test_run.py b/tests/test_run.py index aea5808..d42bc0f 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -247,7 +247,7 @@ def test_run_eynollah_mbreorder_directory(tmp_path, subtests, pytestconfig, capl def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.xml') - outrenderfile = tmp_path.joinpath('render').joinpath('kant_aufklaerung_1784_0020.xml') + outrenderfile = tmp_path.joinpath('render').joinpath('kant_aufklaerung_1784_0020.png') outrenderfile.parent.mkdir() args = [ '-m', MODELS_OCR, From 42fb452a7ea60fab52997ca0f9e58a755b1de08b Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 26 Sep 2025 12:54:29 +0200 Subject: [PATCH 235/374] disable the -doit OCR test --- tests/test_run.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index d42bc0f..b8baf7b 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -264,8 +264,10 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): return logrec.name == 'eynollah' runner = CliRunner() for options in [ - [], # defaults - ["-doit", str(outrenderfile.parent)], + # kba Fri Sep 26 12:53:49 CEST 2025 + # disabled until error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged + # [], # defaults + # ["-doit", str(outrenderfile.parent)], ["-trocr"], ]: with subtests.test(#msg="test CLI", From eb8d4573a823c7674c524b1b19d37fc5d9b062e9 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 26 Sep 2025 13:57:08 +0200 Subject: [PATCH 236/374] tests: also disable ...ocr_directory test --- tests/test_run.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_run.py b/tests/test_run.py index b8baf7b..40ec6cc 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,5 +1,6 @@ from os import environ from pathlib import Path +import pytest import logging from PIL import Image from eynollah.cli import ( @@ -265,7 +266,7 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): runner = CliRunner() for options in [ # kba Fri Sep 26 12:53:49 CEST 2025 - # disabled until error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged + # Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged # [], # defaults # ["-doit", str(outrenderfile.parent)], ["-trocr"], @@ -288,6 +289,7 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): assert len(out_texts) >= 2, ("result is inaccurate", out_texts) assert sum(map(len, out_texts)) > 100, ("result is inaccurate", out_texts) +@pytest.skip("Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged") def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path From 830cc2c30a3f183f939126ec2bbc4cc264974a8a Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 26 Sep 2025 14:37:04 +0200 Subject: [PATCH 237/374] comment out the offending test outright --- tests/test_run.py | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 40ec6cc..da0455a 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -289,26 +289,27 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): assert len(out_texts) >= 2, ("result is inaccurate", out_texts) assert sum(map(len, out_texts)) > 100, ("result is inaccurate", out_texts) -@pytest.skip("Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged") -def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): - indir = testdir.joinpath('resources') - outdir = tmp_path - args = [ - '-m', MODELS_OCR, - '-di', str(indir), - '-dx', str(indir), - '-o', str(outdir), - ] - if pytestconfig.getoption('verbose') > 0: - args.extend(['-l', 'DEBUG']) - caplog.set_level(logging.INFO) - def only_eynollah(logrec): - return logrec.name == 'eynollah' - runner = CliRunner() - with caplog.filtering(only_eynollah): - result = runner.invoke(ocr_cli, args, catch_exceptions=False) - assert result.exit_code == 0, result.stdout - logmsgs = [logrec.message for logrec in caplog.records] - # FIXME: ocr has no logging! - #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs - assert len(list(outdir.iterdir())) == 2 +# kba Fri Sep 26 12:53:49 CEST 2025 +# Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged +# def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): +# indir = testdir.joinpath('resources') +# outdir = tmp_path +# args = [ +# '-m', MODELS_OCR, +# '-di', str(indir), +# '-dx', str(indir), +# '-o', str(outdir), +# ] +# if pytestconfig.getoption('verbose') > 0: +# args.extend(['-l', 'DEBUG']) +# caplog.set_level(logging.INFO) +# def only_eynollah(logrec): +# return logrec.name == 'eynollah' +# runner = CliRunner() +# with caplog.filtering(only_eynollah): +# result = runner.invoke(ocr_cli, args, catch_exceptions=False) +# assert result.exit_code == 0, result.stdout +# logmsgs = [logrec.message for logrec in caplog.records] +# # FIXME: ocr has no logging! +# #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs +# assert len(list(outdir.iterdir())) == 2 From 3123add815f4fc610f90ade8bb5dc9ad6bd634c5 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 26 Sep 2025 15:07:32 +0200 Subject: [PATCH 238/374] :memo: update README --- README.md | 55 ++++++++++++++++++++------------ docs/models.md | 20 +++++++++++- docs/train.md | 81 +++++++++++++++++++++++++++++++++++++---------- tests/test_run.py | 47 ++++++++++++++------------- 4 files changed, 141 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 1adc3d7..4683eb7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Eynollah -> Document Layout Analysis with Deep Learning and Heuristics + +> Document Layout Analysis, Binarization and OCR with Deep Learning and Heuristics [![PyPI Version](https://img.shields.io/pypi/v/eynollah)](https://pypi.org/project/eynollah/) [![GH Actions Test](https://github.com/qurator-spk/eynollah/actions/workflows/test-eynollah.yml/badge.svg)](https://github.com/qurator-spk/eynollah/actions/workflows/test-eynollah.yml) @@ -23,6 +24,7 @@ historical documents and therefore processing can be very slow. We aim to improve this, but contributions are welcome. ## Installation + Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported. For (limited) GPU support the CUDA toolkit needs to be installed. @@ -42,19 +44,30 @@ cd eynollah; pip install -e . Alternatively, you can run `make install` or `make install-dev` for editable installation. +To also install the dependencies for the OCR engines: + +``` +pip install "eynollah[OCR]" +# or +make install EXTRAS=OCR +``` + ## Models -Pretrained models can be downloaded from [qurator-data.de](https://qurator-data.de/eynollah/) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). +Pretrained models can be downloaded from [zenodo](https://zenodo.org/records/17194824) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). ## Train + In case you want to train your own model with Eynollah, have a look at [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md). ## Usage -Eynollah supports four use cases: layout analysis (segmentation), binarization, text recognition (OCR), -and (trainable) reading order detection. + +Eynollah supports five use cases: layout analysis (segmentation), binarization, +image enhancement, text recognition (OCR), and (trainable) reading order detection. ### Layout Analysis + The layout analysis module is responsible for detecting layouts, identifying text lines, and determining reading order using both heuristic methods or a machine-based reading order detection model. @@ -97,58 +110,54 @@ and marginals). The best output quality is produced when RGB images are used as input rather than greyscale or binarized images. ### Binarization + The binarization module performs document image binarization using pretrained pixelwise segmentation models. The command-line interface for binarization of single image can be called like this: ```sh eynollah binarization \ + -i | -di \ + -o \ -m \ - \ - -``` - -and for flowing from a directory like this: - -```sh -eynollah binarization \ - -m \ - -di \ - -do ``` ### OCR + The OCR module performs text recognition from images using two main families of pretrained models: CNN-RNN–based OCR and Transformer-based OCR. The command-line interface for ocr can be called like this: ```sh eynollah ocr \ - -m | --model_name \ -i | -di \ -dx \ - -o + -o \ + -m | --model_name \ ``` ### Machine-based-reading-order + The machine-based reading-order module employs a pretrained model to identify the reading order from layouts represented in PAGE-XML files. The command-line interface for machine based reading order can be called like this: ```sh eynollah machine-based-reading-order \ - -m \ + -i | -di \ -xml | -dx \ + -m \ -o ``` #### Use as OCR-D processor + Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) [processor](https://ocr-d.de/en/spec/cli), formally described in [`ocrd-tool.json`](https://github.com/qurator-spk/eynollah/tree/main/src/eynollah/ocrd-tool.json). In this case, the source image file group with (preferably) RGB images should be used as input like this: - ocrd-eynollah-segment -I OCR-D-IMG -O OCR-D-SEG -P models 2022-04-05 + ocrd-eynollah-segment -I OCR-D-IMG -O OCR-D-SEG -P models eynollah_layout_v0_5_0 If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynollah behaves as follows: - existing regions are kept and ignored (i.e. in effect they might overlap segments from Eynollah results) @@ -160,14 +169,20 @@ If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynol (because some other preprocessing step was in effect like `denoised`), then the output PAGE-XML will be based on that as new top-level (`@imageFilename`) - ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models 2022-04-05 + ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_5_0 Still, in general, it makes more sense to add other workflow steps **after** Eynollah. +There is also an OCR-D processor for the binarization: + + ocrd-sbb-binarize -I OCR-D-IMG -O OCR-D-BIN -P models default-2021-03-09 + #### Additional documentation + Please check the [wiki](https://github.com/qurator-spk/eynollah/wiki). ## How to cite + If you find this tool useful in your work, please consider citing our paper: ```bibtex diff --git a/docs/models.md b/docs/models.md index ac563b0..3d296d5 100644 --- a/docs/models.md +++ b/docs/models.md @@ -1,5 +1,6 @@ # Models documentation -This suite of 14 models presents a document layout analysis (DLA) system for historical documents implemented by + +This suite of 15 models presents a document layout analysis (DLA) system for historical documents implemented by pixel-wise segmentation using a combination of a ResNet50 encoder with various U-Net decoders. In addition, heuristic methods are applied to detect marginals and to determine the reading order of text regions. @@ -23,6 +24,7 @@ See the flowchart below for the different stages and how they interact: ## Models ### Image enhancement + Model card: [Image Enhancement](https://huggingface.co/SBB/eynollah-enhancement) This model addresses image resolution, specifically targeting documents with suboptimal resolution. In instances where @@ -30,12 +32,14 @@ the detection of document layout exhibits inadequate performance, the proposed e the quality and clarity of the images, thus facilitating enhanced visual interpretation and analysis. ### Page extraction / border detection + Model card: [Page Extraction/Border Detection](https://huggingface.co/SBB/eynollah-page-extraction) A problem that can negatively affect OCR are black margins around a page caused by document scanning. A deep learning model helps to crop to the page borders by using a pixel-wise segmentation method. ### Column classification + Model card: [Column Classification](https://huggingface.co/SBB/eynollah-column-classifier) This model is a trained classifier that recognizes the number of columns in a document by use of a training set with @@ -43,6 +47,7 @@ manual classification of all documents into six classes with either one, two, th respectively. ### Binarization + Model card: [Binarization](https://huggingface.co/SBB/eynollah-binarization) This model is designed to tackle the intricate task of document image binarization, which involves segmentation of the @@ -52,6 +57,7 @@ capability of the model enables improved accuracy and reliability in subsequent enhanced document understanding and interpretation. ### Main region detection + Model card: [Main Region Detection](https://huggingface.co/SBB/eynollah-main-regions) This model has employed a different set of labels, including an artificial class specifically designed to encompass the @@ -61,6 +67,7 @@ during the inference phase. By incorporating this methodology, improved efficien model's ability to accurately identify and classify text regions within documents. ### Main region detection (with scaling augmentation) + Model card: [Main Region Detection (with scaling augmentation)](https://huggingface.co/SBB/eynollah-main-regions-aug-scaling) Utilizing scaling augmentation, this model leverages the capability to effectively segment elements of extremely high or @@ -69,12 +76,14 @@ categorizing and isolating such elements, thereby enhancing its overall performa documents with varying scale characteristics. ### Main region detection (with rotation augmentation) + Model card: [Main Region Detection (with rotation augmentation)](https://huggingface.co/SBB/eynollah-main-regions-aug-rotation) This model takes advantage of rotation augmentation. This helps the tool to segment the vertical text regions in a robust way. ### Main region detection (ensembled) + Model card: [Main Region Detection (ensembled)](https://huggingface.co/SBB/eynollah-main-regions-ensembled) The robustness of this model is attained through an ensembling technique that combines the weights from various epochs. @@ -82,16 +91,19 @@ By employing this approach, the model achieves a high level of resilience and st strengths of multiple epochs to enhance its overall performance and deliver consistent and reliable results. ### Full region detection (1,2-column documents) + Model card: [Full Region Detection (1,2-column documents)](https://huggingface.co/SBB/eynollah-full-regions-1column) This model deals with documents comprising of one and two columns. ### Full region detection (3,n-column documents) + Model card: [Full Region Detection (3,n-column documents)](https://huggingface.co/SBB/eynollah-full-regions-3pluscolumn) This model is responsible for detecting headers and drop capitals in documents with three or more columns. ### Textline detection + Model card: [Textline Detection](https://huggingface.co/SBB/eynollah-textline) The method for textline detection combines deep learning and heuristics. In the deep learning part, an image-to-image @@ -106,6 +118,7 @@ segmentation is first deskewed and then the textlines are separated with the sam textline bounding boxes. Later, the strap is rotated back into its original orientation. ### Textline detection (light) + Model card: [Textline Detection Light (simpler but faster method)](https://huggingface.co/SBB/eynollah-textline_light) The method for textline detection combines deep learning and heuristics. In the deep learning part, an image-to-image @@ -119,6 +132,7 @@ enhancing the model's ability to accurately identify and delineate individual te eliminates the need for additional heuristics in extracting textline contours. ### Table detection + Model card: [Table Detection](https://huggingface.co/SBB/eynollah-tables) The objective of this model is to perform table segmentation in historical document images. Due to the pixel-wise @@ -128,17 +142,21 @@ effectively identify and delineate tables within the historical document images, enabling subsequent analysis and interpretation. ### Image detection + Model card: [Image Detection](https://huggingface.co/SBB/eynollah-image-extraction) This model is used for the task of illustration detection only. ### Reading order detection + Model card: [Reading Order Detection]() TODO ## Heuristic methods + Additionally, some heuristic methods are employed to further improve the model predictions: + * After border detection, the largest contour is determined by a bounding box, and the image cropped to these coordinates. * For text region detection, the image is scaled up to make it easier for the model to detect background space between text regions. * A minimum area is defined for text regions in relation to the overall image dimensions, so that very small regions that are noise can be filtered out. diff --git a/docs/train.md b/docs/train.md index 9f44a63..47ad67b 100644 --- a/docs/train.md +++ b/docs/train.md @@ -1,4 +1,5 @@ # Training documentation + This aims to assist users in preparing training datasets, training models, and performing inference with trained models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order detection. For each use case, we provide guidance on how to generate the corresponding training dataset. @@ -11,6 +12,7 @@ The following three tasks can all be accomplished using the code in the * inference with the trained model ## Generate training dataset + The script `generate_gt_for_training.py` is used for generating training datasets. As the results of the following command demonstrates, the dataset generator provides three different commands: @@ -23,14 +25,19 @@ These three commands are: * pagexml2label ### image-enhancement + Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of high-resolution images. The training dataset can then be generated using the following command: -`python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded -images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs -"degrading scales json file"` +```sh +python generate_gt_for_training.py image-enhancement \ + -dis "dir of high resolution images" \ + -dois "dir where degraded images will be written" \ + -dols "dir where the corresponding high resolution image will be written as label" \ + -scs "degrading scales json file" +``` -The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are +The scales JSON file is a dictionary with a key named `scales` and values representing scales smaller than 1. Images are downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose resolution at different scales. The degraded images are used as input images, and the original high-resolution images serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: @@ -42,6 +49,7 @@ serve as labels. The enhancement model can be trained with this generated datase ``` ### machine-based-reading-order + For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's input is a three-channel image: the first and last channels contain information about each of the two text regions, while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. @@ -52,10 +60,18 @@ For output images, it is necessary to specify the width and height. Additionally to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area to the image area, with a default value of zero. To run the dataset generator, use the following command: -`python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images -will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` +```shell +python generate_gt_for_training.py machine-based-reading-order \ + -dx "dir of GT xml files" \ + -domi "dir where output images will be written" \ + -docl "dir where the labels will be written" \ + -ih "height" \ + -iw "width" \ + -min "min area ratio" +``` ### pagexml2label + pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script @@ -119,9 +135,13 @@ graphic region, "stamp" has its own class, while all other types are classified region" are also present in the label. However, other regions like "noise region" and "table region" will not be included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will -be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just -to visualise the labels" "` +```sh +python generate_gt_for_training.py pagexml2label \ + -dx "dir of GT xml files" \ + -do "dir where output label png files will be written" \ + -cfg "custom config json file" \ + -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" +``` We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, @@ -169,12 +189,19 @@ in this scenario, since cropping will be applied to the label files, the directo provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels required for training are obtained. The command should resemble the following: -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will -be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just -to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will -be written" ` +```sh +python generate_gt_for_training.py pagexml2label \ + -dx "dir of GT xml files" \ + -do "dir where output label png files will be written" \ + -cfg "custom config json file" \ + -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" \ + -ps \ + -di "dir where the org images are located" \ + -doi "dir where the cropped output images will be written" +``` ## Train a model + ### classification For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, @@ -225,7 +252,9 @@ And the "dir_eval" the same structure as train directory: The classification model can be trained using the following command line: -`python train.py with config_classification.json` +```sh +python train.py with config_classification.json +``` As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, @@ -276,6 +305,7 @@ The classification model can be trained like the classification case command lin ### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement #### Parameter configuration for segmentation or enhancement usecases + The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. @@ -355,6 +385,7 @@ command, similar to the process for classification and reading order: `python train.py with config_classification.json` #### Binarization + An example config json file for binarization can be like this: ```yaml @@ -550,6 +581,7 @@ For page segmentation (or printspace or border segmentation), the model needs to hence the patches parameter should be set to false. #### layout segmentation + An example config json file for layout segmentation with 5 classes (including background) can be like this: ```yaml @@ -605,26 +637,41 @@ An example config json file for layout segmentation with 5 classes (including ba ## Inference with the trained model ### classification + For conducting inference with a trained model, you simply need to execute the following command line, specifying the directory of the model and the image on which to perform inference: -`python inference.py -m "model dir" -i "image" ` +```sh +python inference.py -m "model dir" -i "image" +``` This will straightforwardly return the class of the image. ### machine based reading order + To infer the reading order using a reading order model, we need a page XML file containing layout information but without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The new XML file with the added reading order will be written to the output directory with the same name. We need to run: -`python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` +```sh +python inference.py \ + -m "model dir" \ + -xml "page xml file" \ + -o "output dir to write new xml with reading order" +``` ### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: -`python inference.py -m "model dir" -i "image" -p -s "output image" ` +```sh +python inference.py \ + -m "model dir" \ + -i "image" \ + -p \ + -s "output image" +``` Note that in the case of page extraction the -p flag is not needed. diff --git a/tests/test_run.py b/tests/test_run.py index da0455a..be928a0 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -289,27 +289,26 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): assert len(out_texts) >= 2, ("result is inaccurate", out_texts) assert sum(map(len, out_texts)) > 100, ("result is inaccurate", out_texts) -# kba Fri Sep 26 12:53:49 CEST 2025 -# Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged -# def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): -# indir = testdir.joinpath('resources') -# outdir = tmp_path -# args = [ -# '-m', MODELS_OCR, -# '-di', str(indir), -# '-dx', str(indir), -# '-o', str(outdir), -# ] -# if pytestconfig.getoption('verbose') > 0: -# args.extend(['-l', 'DEBUG']) -# caplog.set_level(logging.INFO) -# def only_eynollah(logrec): -# return logrec.name == 'eynollah' -# runner = CliRunner() -# with caplog.filtering(only_eynollah): -# result = runner.invoke(ocr_cli, args, catch_exceptions=False) -# assert result.exit_code == 0, result.stdout -# logmsgs = [logrec.message for logrec in caplog.records] -# # FIXME: ocr has no logging! -# #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs -# assert len(list(outdir.iterdir())) == 2 +@pytest.mark.skip("Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged") +def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): + indir = testdir.joinpath('resources') + outdir = tmp_path + args = [ + '-m', MODELS_OCR, + '-di', str(indir), + '-dx', str(indir), + '-o', str(outdir), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'eynollah' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(ocr_cli, args, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + # FIXME: ocr has no logging! + #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs + assert len(list(outdir.iterdir())) == 2 From 37e64b4e458613a433f4837a120a66378ea6668a Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 26 Sep 2025 16:19:04 +0200 Subject: [PATCH 239/374] :memo: changelog --- CHANGELOG.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad86fe5..a05919e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,13 @@ Versioned according to [Semantic Versioning](http://semver.org/). Fixed: - * restoring the contour in the original image caused an error due to an empty tuple + * restoring the contour in the original image caused an error due to an empty tuple, #154 + +Added: + + * `eynollah machine-based-reading-order` CLI to run reading order detection, #175 + * `eynollah enhancement` CLI to run image enhancement, #175 + * Improved models for page extraction and reading order detection, #175 ## [0.4.0] - 2025-04-07 From 6ea6a6280165ff040b63abe8dc9e917b4150a40b Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 26 Sep 2025 16:23:46 +0200 Subject: [PATCH 240/374] :memo: v0.5.0 --- CHANGELOG.md | 4 ++++ src/eynollah/ocrd-tool.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a05919e..0ad9a09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +## [0.5.0] - 2025-09-26 + Fixed: * restoring the contour in the original image caused an error due to an empty tuple, #154 @@ -193,6 +195,8 @@ Fixed: Initial release +[0.5.0]: ../../compare/v0.5.0...v0.4.0 +[0.4.0]: ../../compare/v0.4.0...v0.3.1 [0.3.1]: ../../compare/v0.3.1...v0.3.0 [0.3.0]: ../../compare/v0.3.0...v0.2.0 [0.2.0]: ../../compare/v0.2.0...v0.1.0 diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index fbc6c1a..5d89c92 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -1,5 +1,5 @@ { - "version": "0.4.0", + "version": "0.5.0", "git_url": "https://github.com/qurator-spk/eynollah", "dockerhub": "ocrd/eynollah", "tools": { From 92c1e824dc0683fc74eaa037cabcdb41f49cf677 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Fri, 26 Sep 2025 23:05:47 +0200 Subject: [PATCH 241/374] CD: master is now main --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index d77958b..d2869ed 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -2,7 +2,7 @@ name: CD on: push: - branches: [ "master" ] + branches: [ "main" ] workflow_dispatch: # run manually jobs: From a48e52c00eef1b1e8c85b25bf4d95e46ecaf0cf1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky <38561704+bertsky@users.noreply.github.com> Date: Mon, 29 Sep 2025 13:49:18 +0200 Subject: [PATCH 242/374] :memo: extend changelog for v0.5.0 --- CHANGELOG.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad9a09..bfdd1ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,37 @@ Fixed: * restoring the contour in the original image caused an error due to an empty tuple, #154 +Changed + + * CLIs: read only allowed filename suffixes (image or XML) with `--dir_in` + * CLIs: make all output option required, and `-i` / `-di` required but mutually exclusive + * ocr CLI: drop redundant `-brb` in favour of just `-dib` + * APIs: move all input/output path options from class (kwarg and attribute) ro `run` kwarg + * layout textlines: polygonal also without `-cl` + Added: * `eynollah machine-based-reading-order` CLI to run reading order detection, #175 * `eynollah enhancement` CLI to run image enhancement, #175 * Improved models for page extraction and reading order detection, #175 +Merged PRs: + + * better machine based reading order + layout and textline + ocr by @vahidrezanezhad in https://github.com/qurator-spk/eynollah/pull/175 + * CI: pypi by @kba in https://github.com/qurator-spk/eynollah/pull/154 + * CI: Use most recent actions/setup-python@v5 by @kba in https://github.com/qurator-spk/eynollah/pull/157 + * update docker by @bertsky in https://github.com/qurator-spk/eynollah/pull/159 + * Ocrd fixes by @kba in https://github.com/qurator-spk/eynollah/pull/167 + * Updating readme for eynollah use cases cli by @kba in https://github.com/qurator-spk/eynollah/pull/166 + * OCR-D processor: expose reading_order_machine_based by @bertsky in https://github.com/qurator-spk/eynollah/pull/171 + * prepare release v0.5.0: fix logging by @bertsky in https://github.com/qurator-spk/eynollah/pull/180 + * mb_ro_on_layout: remove copy-pasta code not actually used by @kba in https://github.com/qurator-spk/eynollah/pull/181 + * prepare release v0.5.0: improve CLI docstring, refactor I/O path options from class to run kwargs, increase test coverage @bertsky in #182 + * prepare release v0.5.0: fix for OCR doit subtest by @bertsky in https://github.com/qurator-spk/eynollah/pull/183 + * Prepare release v0.5.0 by @kba in https://github.com/qurator-spk/eynollah/pull/178 + * updating eynollah README, how to use it for use cases by @vahidrezanezhad in https://github.com/qurator-spk/eynollah/pull/156 + * add feedback to command line interface by @michalbubula in https://github.com/qurator-spk/eynollah/pull/170 + ## [0.4.0] - 2025-04-07 Fixed: From 56c4b7af8872514527965c0249553771fa4417d5 Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 14:59:41 +0200 Subject: [PATCH 243/374] :memo: align pre-merge docs/train.md with former upstream train.md syntactically --- docs/train.md | 167 ++++++++++++++++++++++++++----------------------- train/train.md | 135 ++++++++++++++++++++++++++++----------- 2 files changed, 187 insertions(+), 115 deletions(-) diff --git a/docs/train.md b/docs/train.md index 47ad67b..b920a07 100644 --- a/docs/train.md +++ b/docs/train.md @@ -1,10 +1,12 @@ # Training documentation -This aims to assist users in preparing training datasets, training models, and performing inference with trained models. -We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based -reading order detection. For each use case, we provide guidance on how to generate the corresponding training dataset. +This aims to assist users in preparing training datasets, training models, and +performing inference with trained models. We cover various use cases including +pixel-wise segmentation, image classification, image enhancement, and +machine-based reading order detection. For each use case, we provide guidance +on how to generate the corresponding training dataset. -The following three tasks can all be accomplished using the code in the +The following three tasks can all be accomplished using the code in the [`train`](https://github.com/qurator-spk/sbb_pixelwise_segmentation/tree/unifying-training-models) directory: * generate training dataset @@ -13,7 +15,7 @@ The following three tasks can all be accomplished using the code in the ## Generate training dataset -The script `generate_gt_for_training.py` is used for generating training datasets. As the results of the following +The script `generate_gt_for_training.py` is used for generating training datasets. As the results of the following command demonstrates, the dataset generator provides three different commands: `python generate_gt_for_training.py --help` @@ -26,7 +28,7 @@ These three commands are: ### image-enhancement -Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of +Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of high-resolution images. The training dataset can then be generated using the following command: ```sh @@ -37,9 +39,9 @@ python generate_gt_for_training.py image-enhancement \ -scs "degrading scales json file" ``` -The scales JSON file is a dictionary with a key named `scales` and values representing scales smaller than 1. Images are -downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose -resolution at different scales. The degraded images are used as input images, and the original high-resolution images +The scales JSON file is a dictionary with a key named `scales` and values representing scales smaller than 1. Images are +downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose +resolution at different scales. The degraded images are used as input images, and the original high-resolution images serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: ```yaml @@ -50,14 +52,14 @@ serve as labels. The enhancement model can be trained with this generated datase ### machine-based-reading-order -For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's -input is a three-channel image: the first and last channels contain information about each of the two text regions, -while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. -To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct +For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's +input is a three-channel image: the first and last channels contain information about each of the two text regions, +while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. +To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct reading order. -For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set -to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area +For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set +to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area to the image area, with a default value of zero. To run the dataset generator, use the following command: ```shell @@ -74,15 +76,15 @@ python generate_gt_for_training.py machine-based-reading-order \ pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. -To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script -expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled -as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four +To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script +expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled +as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. -In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired +In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired element is automatically encoded as 1 in the PNG label. -To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. +To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. For example, in the case of 'textline' detection, the JSON file would resemble this: ```yaml @@ -116,23 +118,23 @@ A possible custom config json file for layout segmentation where the "printspace } ``` -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. -In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. -For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', +For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. +In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. +For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', and 'tableregion'. -Text regions and graphic regions also have their own specific types. The known types for text regions are 'paragraph', -'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', -and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and +Text regions and graphic regions also have their own specific types. The known types for text regions are 'paragraph', +'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', +and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and 'signature'. -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined -two additional types, "rest_as_paragraph" and "rest_as_decoration", to ensure that no unknown types are missed. +Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined +two additional types, "rest_as_paragraph" and "rest_as_decoration", to ensure that no unknown types are missed. This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown -as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the -graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator -region" are also present in the label. However, other regions like "noise region" and "table region" will not be +In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown +as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the +graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator +region" are also present in the label. However, other regions like "noise region" and "table region" will not be included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. ```sh @@ -143,8 +145,8 @@ python generate_gt_for_training.py pagexml2label \ -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" ``` -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key -is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, +We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key +is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, the example JSON config file should look like this: ```yaml @@ -167,13 +169,13 @@ the example JSON config file should look like this: } ``` -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the +This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the elements labeled as "paragraph," "header," "heading," and "marginalia." -For "textline", "word", and "glyph", the artificial class on the boundaries will be activated only if the -"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements -represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the -artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use +For "textline", "word", and "glyph", the artificial class on the boundaries will be activated only if the +"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements +represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the +artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use case: ```yaml @@ -183,10 +185,10 @@ case: } ``` -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to -crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that -in this scenario, since cropping will be applied to the label files, the directory of the original images must be -provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels +If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to +crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that +in this scenario, since cropping will be applied to the label files, the directory of the original images must be +provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels required for training are obtained. The command should resemble the following: ```sh @@ -204,11 +206,11 @@ python generate_gt_for_training.py pagexml2label \ ### classification -For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, -all we require is a training directory with subdirectories, each containing images of its respective classes. We need -separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both -directories. Additionally, the class names should be specified in the config JSON file, as shown in the following -example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the +For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, +all we require is a training directory with subdirectories, each containing images of its respective classes. We need +separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both +directories. Additionally, the class names should be specified in the config JSON file, as shown in the following +example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the "classification_classes_name" key in the config file should appear as follows: ```yaml @@ -233,7 +235,7 @@ example. If, for instance, we aim to classify "apple" and "orange," with a total The "dir_train" should be like this: -``` +``` . └── train # train directory ├── apple # directory of images for apple class @@ -242,7 +244,7 @@ The "dir_train" should be like this: And the "dir_eval" the same structure as train directory: -``` +``` . └── eval # evaluation directory ├── apple # directory of images for apple class @@ -256,9 +258,9 @@ The classification model can be trained using the following command line: python train.py with config_classification.json ``` -As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. -This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, -an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". +As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. +This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, +an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". ### reading order @@ -306,25 +308,25 @@ The classification model can be trained like the classification case command lin #### Parameter configuration for segmentation or enhancement usecases -The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, -its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for +The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, +its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we -* offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first +* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we +* offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first * apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. * task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this -* parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be +* patches: If you want to break input images into smaller patches (input size of the model) you need to set this +* parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be * set to ``false``. * n_batch: Number of batches at each iteration. -* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it +* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it * should set to 1. And for the case of layout detection just the unique number of classes should be given. * n_epochs: Number of epochs. * input_height: This indicates the height of model's input. * input_width: This indicates the width of model's input. * weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved +* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved * in a folder named "pretrained_model" in the same directory of "train.py" script. * augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. * flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. @@ -344,9 +346,15 @@ classification and machine-based reading order, as you can see in their example * brightness: The amount of brightenings. * thetha: Rotation angles. * degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. +* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the + training. So it is needed to providethe dir of trained model with "dir_of_start_model" and index for naming + themodels. For example if you have already trained for 3 epochs then your lastindex is 2 and if you want to continue + from model_1.h5, you can set +``index_start`` to 3 to start naming model with index 3. * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train + and eval data are in"dir_output".Since when once we provide training data we resize and augmentthem and then wewrite + them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. * index_start: Starting index for saved models in the case that "continue_training" is ``true``. * dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. @@ -379,7 +387,7 @@ And the "dir_eval" the same structure as train directory: └── labels # directory of labels ``` -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following +After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following command, similar to the process for classification and reading order: `python train.py with config_classification.json` @@ -429,7 +437,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -474,7 +482,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -519,7 +527,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -529,7 +537,7 @@ An example config json file for binarization can be like this: } ``` -It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel +It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel image. #### Page extraction @@ -567,7 +575,7 @@ image. "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -577,8 +585,8 @@ image. } ``` -For page segmentation (or printspace or border segmentation), the model needs to view the input image in its entirety, -hence the patches parameter should be set to false. +For page segmentation (or print space or border segmentation), the model needs to view the input image in its +entirety,hence the patches parameter should be set to false. #### layout segmentation @@ -625,7 +633,7 @@ An example config json file for layout segmentation with 5 classes (including ba "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -638,7 +646,7 @@ An example config json file for layout segmentation with 5 classes (including ba ### classification -For conducting inference with a trained model, you simply need to execute the following command line, specifying the +For conducting inference with a trained model, you simply need to execute the following command line, specifying the directory of the model and the image on which to perform inference: ```sh @@ -649,10 +657,9 @@ This will straightforwardly return the class of the image. ### machine based reading order -To infer the reading order using a reading order model, we need a page XML file containing layout information but -without the reading order. We simply need to provide the model directory, the XML file, and the output directory. -The new XML file with the added reading order will be written to the output directory with the same name. -We need to run: +To infer the reading order using a reading order model, we need a page XML file containing layout information but +without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The +new XML file with the added reading order will be written to the output directory with the same name. We need to run: ```sh python inference.py \ @@ -662,8 +669,8 @@ python inference.py \ ``` ### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement -For conducting inference with a trained model for segmentation and enhancement you need to run the following command -line: + +For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: ```sh python inference.py \ @@ -675,5 +682,5 @@ python inference.py \ Note that in the case of page extraction the -p flag is not needed. -For segmentation or binarization tasks, if a ground truth (GT) label is available, the IoU evaluation metric can be +For segmentation or binarization tasks, if a ground truth (GT) label is available, the IoU evaluation metric can be calculated for the output. To do this, you need to provide the GT label using the argument -gt. diff --git a/train/train.md b/train/train.md index 553522b..3eeb715 100644 --- a/train/train.md +++ b/train/train.md @@ -1,6 +1,9 @@ # Documentation for Training Models -This repository assists users in preparing training datasets, training models, and performing inference with trained models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training dataset. +This repository assists users in preparing training datasets, training models, and performing inference with trained +models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and +machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training +dataset. All these use cases are now utilized in the Eynollah workflow. As mentioned, the following three tasks can be accomplished using this repository: @@ -23,11 +26,15 @@ These three commands are: ### image-enhancement -Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of high-resolution images. The training dataset can then be generated using the following command: +Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of +high-resolution images. The training dataset can then be generated using the following command: `python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs "degrading scales json file"` -The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose resolution at different scales. The degraded images are used as input images, and the original high-resolution images serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: +The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are +downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose +resolution at different scales. The degraded images are used as input images, and the original high-resolution images +serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: ```yaml { @@ -37,21 +44,33 @@ The scales JSON file is a dictionary with a key named 'scales' and values repres ### machine-based-reading-order -For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's input is a three-channel image: the first and last channels contain information about each of the two text regions, while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct reading order. +For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's +input is a three-channel image: the first and last channels contain information about each of the two text regions, +while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. +To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct +reading order. -For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area to the image area, with a default value of zero. To run the dataset generator, use the following command: +For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set +to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area +to the image area, with a default value of zero. To run the dataset generator, use the following command: `python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` ### pagexml2label -pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. -To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. +pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, +including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. +To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script +expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled +as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four +elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. -In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired element is automatically encoded as 1 in the PNG label. +In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired +element is automatically encoded as 1 in the PNG label. -To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. For example, in the case of 'textline' detection, the JSON file would resemble this: +To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. +For example, in the case of 'textline' detection, the JSON file would resemble this: ```yaml { @@ -83,16 +102,31 @@ A possible custom config json file for layout segmentation where the "printspac "printspace_as_class_in_layout" : 8 } ``` -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', and 'tableregion'. +For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a +given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an +image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', +and 'tableregion'. -Text regions and graphic regions also have their own specific types. The known types for us for text regions are 'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and 'signature'. -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. +Text regions and graphic regions also have their own specific types. The known types for us for text regions are +'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', +'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', +'stamp', and 'signature'. -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator region" are also present in the label. However, other regions like "noise region" and "table region" will not be included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. +Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two +additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, +users can extract all known types from the labels and be confident that no unknown types are overlooked. + +In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown +as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the +graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator +region" are also present in the label. However, other regions like "noise region" and "table region" will not be +included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. `python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" "` -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, the example JSON config file should look like this: +We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key +is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, +the example JSON config file should look like this: ```yaml { @@ -114,9 +148,14 @@ We have also defined an artificial class that can be added to the boundary of te } ``` -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the elements labeled as "paragraph," "header," "heading," and "marginalia." +This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the +elements labeled as "paragraph," "header," "heading," and "marginalia." -For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the "artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use case: +For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the +"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements +represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the +artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use +case: ```yaml { @@ -125,7 +164,11 @@ For "textline," "word," and "glyph," the artificial class on the boundaries will } ``` -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that in this scenario, since cropping will be applied to the label files, the directory of the original images must be provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels required for training are obtained. The command should resemble the following: +If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to +crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that +in this scenario, since cropping will be applied to the label files, the directory of the original images must be +provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels +required for training are obtained. The command should resemble the following: `python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will be written" ` @@ -178,7 +221,10 @@ The classification model can be trained using the following command line: `python train.py with config_classification.json` -As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". +As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. +This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, +an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". +Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". ### reading order An example config json file for machine based reading order should be like this: @@ -225,18 +271,25 @@ The classification model can be trained like the classification case command lin #### Parameter configuration for segmentation or enhancement usecases -The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. +The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, +its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for +classification and machine-based reading order, as you can see in their example config files. -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. +* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we + offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first +apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. * task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be set to ``false``. +* patches: If you want to break input images into smaller patches (input size of the model) you need to set this + parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be +set to ``false``. * n_batch: Number of batches at each iteration. * n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. * n_epochs: Number of epochs. * input_height: This indicates the height of model's input. * input_width: This indicates the width of model's input. * weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved in a folder named "pretrained_model" in the same directory of "train.py" script. +* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved + in a folder named "pretrained_model" in the same directory of "train.py" script. * augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. * flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. * blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. @@ -255,9 +308,14 @@ The following parameter configuration can be applied to all segmentation use cas * brightness: The amount of brightenings. * thetha: Rotation angles. * degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. +* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the + training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the + models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from + model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train + and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we + write them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. * index_start: Starting index for saved models in the case that "continue_training" is ``true``. * dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. @@ -290,7 +348,8 @@ And the "dir_eval" the same structure as train directory: └── labels # directory of labels ``` -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following command, similar to the process for classification and reading order: +After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following +command, similar to the process for classification and reading order: `python train.py with config_classification.json` @@ -339,7 +398,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -384,7 +443,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -429,7 +488,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -439,7 +498,8 @@ An example config json file for binarization can be like this: } ``` -It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel image. +It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel +image. #### Page extraction @@ -486,7 +546,8 @@ It's important to mention that the value of n_classes for enhancement should be } ``` -For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, hence the patches parameter should be set to false. +For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, +hence the patches parameter should be set to false. #### layout segmentation @@ -533,7 +594,7 @@ An example config json file for layout segmentation with 5 classes (including ba "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -543,9 +604,11 @@ An example config json file for layout segmentation with 5 classes (including ba } ``` ## Inference with the trained model + ### classification -For conducting inference with a trained model, you simply need to execute the following command line, specifying the directory of the model and the image on which to perform inference: +For conducting inference with a trained model, you simply need to execute the following command line, specifying the +directory of the model and the image on which to perform inference: `python inference.py -m "model dir" -i "image" ` @@ -554,8 +617,9 @@ This will straightforwardly return the class of the image. ### machine based reading order - -To infer the reading order using an reading order model, we need a page XML file containing layout information but without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The new XML file with the added reading order will be written to the output directory with the same name. We need to run: +To infer the reading order using an reading order model, we need a page XML file containing layout information but +without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The +new XML file with the added reading order will be written to the output directory with the same name. We need to run: `python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` @@ -570,7 +634,8 @@ For conducting inference with a trained model for segmentation and enhancement y Note that in the case of page extraction the -p flag is not needed. -For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be calculated for the output. To do this, you need to provide the GT label using the argument -gt. +For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be +calculated for the output. To do this, you need to provide the GT label using the argument -gt. From ea05461dfeb9551f2e333d03a708e01295ccfb2d Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 15:04:46 +0200 Subject: [PATCH 244/374] add documentation on eynollah layout from eynollah wiki --- docs/eynollah-layout.md | 100 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 docs/eynollah-layout.md diff --git a/docs/eynollah-layout.md b/docs/eynollah-layout.md new file mode 100644 index 0000000..e76ed51 --- /dev/null +++ b/docs/eynollah-layout.md @@ -0,0 +1,100 @@ +# `eynollah layout` documentation + +Eynollah can currently be used to detect the following region types/elements: +* Background +* [Border](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_BorderType.html) +* [Textregion](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextRegionType.html) +* [Textline](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextLineType.html) +* [Header](https://ocr-d.de/en/gt-guidelines/trans/lyUeberschrift.html) +* [Image](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_ImageRegionType.html) +* [Separator](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_SeparatorRegionType.html) +* [Marginalia](https://ocr-d.de/en/gt-guidelines/trans/lyMarginalie.html) +* [Initial (Drop Capital)](https://ocr-d.de/en/gt-guidelines/trans/lyInitiale.html) +* [Table](https://ocr-d.de/en/gt-guidelines/trans/lyTabellen.html) + +In addition, the tool can detect the [ReadingOrder](https://ocr-d.de/en/gt-guidelines/trans/lyLeserichtung.html) of text regions, both from left-to-right or from right-to-left. The final goal is to feed the output to an OCR model. + +## Method description + +Eynollah is based on pixelwise segmentation using a combination of a ResNet50 encoder with various U-Net decoders. +It uses a combination of multiple models and heuristics (see the flowchart below for the different stages and how they interact): +* [Border detection](https://github.com/qurator-spk/eynollah#border-detection) +* [Layout detection](https://github.com/qurator-spk/eynollah#layout-detection) +* [Textline detection](https://github.com/qurator-spk/eynollah#textline-detection) +* [Image enhancement](https://github.com/qurator-spk/eynollah#Image_enhancement) +* [Scale classification](https://github.com/qurator-spk/eynollah#Scale_classification) +* [Heuristic methods](https://https://github.com/qurator-spk/eynollah#heuristic-methods) + +![](https://user-images.githubusercontent.com/952378/100619946-1936f680-331e-11eb-9297-6e8b4cab3c16.png) + +### Border detection +For the purpose of text recognition (OCR) and in order to avoid noise being introduced from texts outside the printspace, one first needs to detect the border of the printed frame. This is done by a binary pixel-wise-segmentation model trained on a dataset of 2,000 documents where about 1,200 of them come from the [dhSegment](https://github.com/dhlab-epfl/dhSegment/) project (you can download the dataset from [here](https://github.com/dhlab-epfl/dhSegment/releases/download/v0.2/pages.zip)) and the remainder having been annotated in SBB. For border detection, the model needs to be fed with the whole image at once rather than separated in patches. + +### Layout detection +As a next step, text regions need to be identified by means of layout detection. Again a pixel-wise segmentation model was trained on 131 labeled images from the SBB digital collections, including some data augmentation. Since the target of this tool are historical documents, we consider as main region types text regions, separators, images, tables and background - each with their own subclasses, e.g. in the case of text regions, subclasses like header/heading, drop capital, main body text etc. While it would be desirable to detect and classify each of these classes in a granular way, there are also limitations due to having a suitably large and balanced training set. Accordingly, the current version of this tool is focussed on the main region types background, text region, image and separator. + +### Textline detection +In a subsequent step, binary pixel-wise segmentation is used again to classify pixels in a document that constitute textlines. For textline segmentation, a model was initially trained on documents with only one column/block of text and some augmentation with regard to scaling. By fine-tuning the parameters also for multi-column documents, additional training data was produced that resulted in a much more robust textline detection model. + +### Image enhancement +This is an image to image model which input was low quality of an image and label was actually the original image. For this one we did not have any GT, so we decreased the quality of documents in SBB and then feed them into model. + +### Scale classification +This is simply an image classifier which classifies images based on their scales or better to say based on their number of columns. + +### Heuristic methods +Some heuristic methods are also employed to further improve the model predictions: +* After border detection, the largest contour is determined by a bounding box, and the image cropped to these coordinates. +* For text region detection, the image is scaled up to make it easier for the model to detect background space between text regions. +* A minimum area is defined for text regions in relation to the overall image dimensions, so that very small regions that are noise can be filtered out. +* Deskewing is applied on the text region level (due to regions having different degrees of skew) in order to improve the textline segmentation result. +* After deskewing, a calculation of the pixel distribution on the X-axis allows the separation of textlines (foreground) and background pixels. +* Finally, using the derived coordinates, bounding boxes are determined for each textline. + +## Models + +TODO + +## How to use + +First, this model makes use of up to 9 trained models which are responsible for different operations like size detection, column classification, image enhancement, page extraction, main layout detection, full layout detection and textline detection.That does not mean that all 9 models are always required for every document. Based on the document characteristics and parameters specified, different scenarios can be applied. + +* If none of the parameters is set to `true`, the tool will perform a layout detection of main regions (background, text, images, separators and marginals). An advantage of this tool is that it tries to extract main text regions separately as much as possible. + +* If you set `-ae` (**a**llow image **e**nhancement) parameter to `true`, the tool will first check the ppi (pixel-per-inch) of the image and when it is less than 300, the tool will resize it and only then image enhancement will occur. Image enhancement can also take place without this option, but by setting this option to `true`, the layout xml data (e.g. coordinates) will be based on the resized and enhanced image instead of the original image. + +* For some documents, while the quality is good, their scale is very large, and the performance of tool decreases. In such cases you can set `-as` (**a**llow **s**caling) to `true`. With this option enabled, the tool will try to rescale the image and only then the layout detection process will begin. + +* If you care about drop capitals (initials) and headings, you can set `-fl` (**f**ull **l**ayout) to `true`. With this setting, the tool can currently distinguish 7 document layout classes/elements. + +* In cases where the document includes curved headers or curved lines, rectangular bounding boxes for textlines will not be a great option. In such cases it is strongly recommended setting the flag `-cl` (**c**urved **l**ines) to `true` to find contours of curved lines instead of rectangular bounding boxes. Be advised that enabling this option increases the processing time of the tool. + +* To crop and save image regions inside the document, set the parameter `-si` (**s**ave **i**mages) to true and provide a directory path to store the extracted images. + +* To extract only images from a document, set the parameter `-eoi` (**e**xtract **o**nly **i**mages). Choosing this option disables any other processing. To save the cropped images add `-ep` and `-si`. + +* This tool is actively being developed. If problems occur, or the performance does not meet your expectations, we welcome your feedback via [issues](https://github.com/qurator-spk/eynollah/issues). + + +### `--full-layout` vs `--no-full-layout` + +Here are the difference in elements detected depending on the `--full-layout`/`--no-full-layout` command line flags: + +| | `--full-layout` | `--no-full-layout` | +| --- | --- | --- | +| reading order | x | x | +| header regions | x | - | +| text regions | x | x | +| text regions / text line | x | x | +| drop-capitals | x | - | +| marginals | x | x | +| marginals / text line | x | x | +| image region | x | x | + +### Use as OCR-D processor + +Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) processor. In this case, the source image file group with (preferably) RGB images should be used as input (the image provided by `@imageFilename` is passed on directly): + +`ocrd-eynollah-segment -I OCR-D-IMG -O SEG-LINE -P models` + +## Examples From 52a7c93319d094c47fc1376171ca890cc80f5936 Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 15:05:05 +0200 Subject: [PATCH 245/374] add documentation on training eynollah from sbb_pixelwise_segmentation wiki --- docs/train_wiki.md | 576 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 576 insertions(+) create mode 100644 docs/train_wiki.md diff --git a/docs/train_wiki.md b/docs/train_wiki.md new file mode 100644 index 0000000..d1c0875 --- /dev/null +++ b/docs/train_wiki.md @@ -0,0 +1,576 @@ +# Documentation + +This repository assists users in preparing training datasets, training models, and performing inference with trained models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training dataset. +All these use cases are now utilized in the Eynollah workflow. +As mentioned, the following three tasks can be accomplished using this repository: + +* Generate training dataset +* Train a model +* Inference with the trained model + +## Generate training dataset +The script generate_gt_for_training.py is used for generating training datasets. As the results of the following command demonstrate, the dataset generator provides three different commands: + +`python generate_gt_for_training.py --help` + + +These three commands are: + +* image-enhancement +* machine-based-reading-order +* pagexml2label + + +### image-enhancement + +Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of high-resolution images. The training dataset can then be generated using the following command: + +`python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs "degrading scales json file"` + +The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose resolution at different scales. The degraded images are used as input images, and the original high-resolution images serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: + +```yaml +{ + "scales": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] +} +``` + +### machine-based-reading-order + +For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's input is a three-channel image: the first and last channels contain information about each of the two text regions, while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct reading order. + +For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area to the image area, with a default value of zero. To run the dataset generator, use the following command: + + +`python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` + +### pagexml2label + +pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. +To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. + +In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired element is automatically encoded as 1 in the PNG label. + +To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. For example, in the case of 'textline' detection, the JSON file would resemble this: + +```yaml +{ +"use_case": "textline" +} +``` + +In the case of layout segmentation a possible custom config json file can be like this: + +```yaml +{ +"use_case": "layout", +"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, +"imageregion":4, +"separatorregion":5, +"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} +} +``` + +A possible custom config json file for layout segmentation where the "printspace" is wished to be a class: + +```yaml +{ +"use_case": "layout", +"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, +"imageregion":4, +"separatorregion":5, +"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} +"printspace_as_class_in_layout" : 8 +} +``` +For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', and 'tableregion'. + +Text regions and graphic regions also have their own specific types. The known types for us for text regions are 'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and 'signature'. +Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. + +In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator region" are also present in the label. However, other regions like "noise region" and "table region" will not be included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. + +`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" "` + +We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, the example JSON config file should look like this: + +```yaml +{ + "use_case": "layout", + "textregions": { + "paragraph": 1, + "drop-capital": 1, + "header": 2, + "heading": 2, + "marginalia": 3 + }, + "imageregion": 4, + "separatorregion": 5, + "graphicregions": { + "rest_as_decoration": 6 + }, + "artificial_class_on_boundary": ["paragraph", "header", "heading", "marginalia"], + "artificial_class_label": 7 +} +``` + +This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the elements labeled as "paragraph," "header," "heading," and "marginalia." + +For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the "artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use case: + +```yaml +{ + "use_case": "textline", + "artificial_class_label": 2 +} +``` + +If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that in this scenario, since cropping will be applied to the label files, the directory of the original images must be provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels required for training are obtained. The command should resemble the following: + +`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will be written" ` + +## Train a model +### classification + +For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, all we require is a training directory with subdirectories, each containing images of its respective classes. We need separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both directories. Additionally, the class names should be specified in the config JSON file, as shown in the following example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the "classification_classes_name" key in the config file should appear as follows: + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "classification", + "n_classes" : 2, + "n_epochs" : 10, + "input_height" : 448, + "input_width" : 448, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "f1_threshold_classification": 0.8, + "pretraining" : true, + "classification_classes_name" : {"0":"apple", "1":"orange"}, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +The "dir_train" should be like this: + +``` +. +└── train # train directory + ├── apple # directory of images for apple class + └── orange # directory of images for orange class +``` + +And the "dir_eval" the same structure as train directory: + +``` +. +└── eval # evaluation directory + ├── apple # directory of images for apple class + └── orange # directory of images for orange class + +``` + +The classification model can be trained using the following command line: + +`python train.py with config_classification.json` + + +As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". + +### reading order +An example config json file for machine based reading order should be like this: + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "reading_order", + "n_classes" : 1, + "n_epochs" : 5, + "input_height" : 672, + "input_width" : 448, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "pretraining" : true, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +The "dir_train" should be like this: + +``` +. +└── train # train directory + ├── images # directory of images + └── labels # directory of labels +``` + +And the "dir_eval" the same structure as train directory: + +``` +. +└── eval # evaluation directory + ├── images # directory of images + └── labels # directory of labels +``` + +The classification model can be trained like the classification case command line. + +### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement + +#### Parameter configuration for segmentation or enhancement usecases + +The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. + +* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. +* task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". +* patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be set to ``false``. +* n_batch: Number of batches at each iteration. +* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. +* n_epochs: Number of epochs. +* input_height: This indicates the height of model's input. +* input_width: This indicates the width of model's input. +* weight_decay: Weight decay of l2 regularization of model layers. +* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved in a folder named "pretrained_model" in the same directory of "train.py" script. +* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. +* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. +* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. +* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" parameter. +* degrading: If ``true``, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. +* brightening: If ``true``, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. +* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. +* rotation: If ``true``, 90 degree rotation will be applied on image. +* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. +* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. +* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. +* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. +* flip_index: Type of flips. +* blur_k: Type of blurrings. +* scales: Scales of scaling. +* brightness: The amount of brightenings. +* thetha: Rotation angles. +* degrade_scales: The amount of degradings. +* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. +* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. +* index_start: Starting index for saved models in the case that "continue_training" is ``true``. +* dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. +* transformer_num_patches_xy: Number of patches for vision transformer in x and y direction respectively. +* transformer_patchsize_x: Patch size of vision transformer patches in x direction. +* transformer_patchsize_y: Patch size of vision transformer patches in y direction. +* transformer_projection_dim: Transformer projection dimension. Default value is 64. +* transformer_mlp_head_units: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. +* transformer_layers: transformer layers. Default value is 8. +* transformer_num_heads: Transformer number of heads. Default value is 4. +* transformer_cnn_first: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. + +In the case of segmentation and enhancement the train and evaluation directory should be as following. + +The "dir_train" should be like this: + +``` +. +└── train # train directory + ├── images # directory of images + └── labels # directory of labels +``` + +And the "dir_eval" the same structure as train directory: + +``` +. +└── eval # evaluation directory + ├── images # directory of images + └── labels # directory of labels +``` + +After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following command, similar to the process for classification and reading order: + +`python train.py with config_classification.json` + +#### Binarization + +An example config json file for binarization can be like this: + +```yaml +{ + "backbone_type" : "transformer", + "task": "binarization", + "n_classes" : 2, + "n_epochs" : 4, + "input_height" : 224, + "input_width" : 672, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "transformer_num_patches_xy": [7, 7], + "transformer_patchsize_x": 3, + "transformer_patchsize_y": 1, + "transformer_projection_dim": 192, + "transformer_mlp_head_units": [128, 64], + "transformer_layers": 8, + "transformer_num_heads": 4, + "transformer_cnn_first": true, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +#### Textline + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "segmentation", + "n_classes" : 2, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +#### Enhancement + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "enhancement", + "n_classes" : 3, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 4, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel image. + +#### Page extraction + +```yaml +{ + "backbone_type" : "nontransformer", + "task": "segmentation", + "n_classes" : 2, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : false, + "pretraining" : true, + "augmentation" : false, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` + +For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, hence the patches parameter should be set to false. + +#### layout segmentation + +An example config json file for layout segmentation with 5 classes (including background) can be like this: + +```yaml +{ + "backbone_type" : "transformer", + "task": "segmentation", + "n_classes" : 5, + "n_epochs" : 4, + "input_height" : 448, + "input_width" : 224, + "weight_decay" : 1e-6, + "n_batch" : 1, + "learning_rate": 1e-4, + "patches" : true, + "pretraining" : true, + "augmentation" : true, + "flip_aug" : false, + "blur_aug" : false, + "scaling" : true, + "degrading": false, + "brightening": false, + "binarization" : false, + "scaling_bluring" : false, + "scaling_binarization" : false, + "scaling_flip" : false, + "rotation": false, + "rotation_not_90": false, + "transformer_num_patches_xy": [7, 14], + "transformer_patchsize_x": 1, + "transformer_patchsize_y": 1, + "transformer_projection_dim": 64, + "transformer_mlp_head_units": [128, 64], + "transformer_layers": 8, + "transformer_num_heads": 4, + "transformer_cnn_first": true, + "blur_k" : ["blur","guass","median"], + "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], + "brightness" : [1.3, 1.5, 1.7, 2], + "degrade_scales" : [0.2, 0.4], + "flip_index" : [0, 1, -1], + "thetha" : [10, -10], + "continue_training": false, + "index_start" : 0, + "dir_of_start_model" : " ", + "weighted_loss": false, + "is_loss_soft_dice": false, + "data_is_provided": false, + "dir_train": "./train", + "dir_eval": "./eval", + "dir_output": "./output" +} +``` +## Inference with the trained model +### classification + +For conducting inference with a trained model, you simply need to execute the following command line, specifying the directory of the model and the image on which to perform inference: + + +`python inference.py -m "model dir" -i "image" ` + +This will straightforwardly return the class of the image. + +### machine based reading order + + +To infer the reading order using an reading order model, we need a page XML file containing layout information but without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The new XML file with the added reading order will be written to the output directory with the same name. We need to run: + +`python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` + + +### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement + +For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: + + +`python inference.py -m "model dir" -i "image" -p -s "output image" ` + + +Note that in the case of page extraction the -p flag is not needed. + +For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be calculated for the output. To do this, you need to provide the GT label using the argument -gt. + + + From 6d379782abf6de1574912878a2dc61d2cfa0d18c Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 15:11:02 +0200 Subject: [PATCH 246/374] :memo: align former upstream train.md with wiki train.md syntactically --- docs/train_wiki.md | 145 +++++++++++++++++++++++++++++++++------------ train/train.md | 18 +++--- 2 files changed, 116 insertions(+), 47 deletions(-) diff --git a/docs/train_wiki.md b/docs/train_wiki.md index d1c0875..5158a80 100644 --- a/docs/train_wiki.md +++ b/docs/train_wiki.md @@ -1,7 +1,10 @@ # Documentation -This repository assists users in preparing training datasets, training models, and performing inference with trained models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training dataset. -All these use cases are now utilized in the Eynollah workflow. +This repository assists users in preparing training datasets, training models, and performing inference with trained +models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and +machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training +dataset. +All these use cases are now utilized in the Eynollah workflow. As mentioned, the following three tasks can be accomplished using this repository: * Generate training dataset @@ -23,11 +26,15 @@ These three commands are: ### image-enhancement -Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of high-resolution images. The training dataset can then be generated using the following command: +Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of +high-resolution images. The training dataset can then be generated using the following command: `python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs "degrading scales json file"` -The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose resolution at different scales. The degraded images are used as input images, and the original high-resolution images serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: +The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are +downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose +resolution at different scales. The degraded images are used as input images, and the original high-resolution images +serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: ```yaml { @@ -37,21 +44,34 @@ The scales JSON file is a dictionary with a key named 'scales' and values repres ### machine-based-reading-order -For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's input is a three-channel image: the first and last channels contain information about each of the two text regions, while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct reading order. +For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's +input is a three-channel image: the first and last channels contain information about each of the two text regions, +while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. +To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct +reading order. -For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area to the image area, with a default value of zero. To run the dataset generator, use the following command: +For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set +to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area +to the image area, with a default value of zero. To run the dataset generator, use the following command: `python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` ### pagexml2label -pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. -To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. +pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, +including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. -In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired element is automatically encoded as 1 in the PNG label. +To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script +expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled +as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four +elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. -To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. For example, in the case of 'textline' detection, the JSON file would resemble this: +In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired +element is automatically encoded as 1 in the PNG label. + +To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. +For example, in the case of 'textline' detection, the JSON file would resemble this: ```yaml { @@ -83,16 +103,32 @@ A possible custom config json file for layout segmentation where the "printspac "printspace_as_class_in_layout" : 8 } ``` -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', and 'tableregion'. -Text regions and graphic regions also have their own specific types. The known types for us for text regions are 'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and 'signature'. -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. +For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a +given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an +image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', +and 'tableregion'. -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator region" are also present in the label. However, other regions like "noise region" and "table region" will not be included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. +Text regions and graphic regions also have their own specific types. The known types for us for text regions are +'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', +'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', +'stamp', and 'signature'. + +Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two +additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, +users can extract all known types from the labels and be confident that no unknown types are overlooked. + +In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown +as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the +graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator +region" are also present in the label. However, other regions like "noise region" and "table region" will not be +included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. `python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" "` -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, the example JSON config file should look like this: +We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key +is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, +the example JSON config file should look like this: ```yaml { @@ -114,9 +150,14 @@ We have also defined an artificial class that can be added to the boundary of te } ``` -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the elements labeled as "paragraph," "header," "heading," and "marginalia." +This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the +elements labeled as "paragraph," "header," "heading," and "marginalia." -For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the "artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use case: +For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the +"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements +represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the +artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use +case: ```yaml { @@ -125,7 +166,11 @@ For "textline," "word," and "glyph," the artificial class on the boundaries will } ``` -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that in this scenario, since cropping will be applied to the label files, the directory of the original images must be provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels required for training are obtained. The command should resemble the following: +If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to +crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that +in this scenario, since cropping will be applied to the label files, the directory of the original images must be +provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels +required for training are obtained. The command should resemble the following: `python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will be written" ` @@ -156,7 +201,7 @@ For the classification use case, we haven't provided a ground truth generator, a The "dir_train" should be like this: -``` +``` . └── train # train directory ├── apple # directory of images for apple class @@ -165,7 +210,7 @@ The "dir_train" should be like this: And the "dir_eval" the same structure as train directory: -``` +``` . └── eval # evaluation directory ├── apple # directory of images for apple class @@ -178,7 +223,10 @@ The classification model can be trained using the following command line: `python train.py with config_classification.json` -As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". +As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. +This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, +an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". +Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". ### reading order An example config json file for machine based reading order should be like this: @@ -225,18 +273,25 @@ The classification model can be trained like the classification case command lin #### Parameter configuration for segmentation or enhancement usecases -The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. +The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, +its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for +classification and machine-based reading order, as you can see in their example config files. -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. +* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we + offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first + apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. * task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be set to ``false``. +* patches: If you want to break input images into smaller patches (input size of the model) you need to set this + parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be + set to ``false``. * n_batch: Number of batches at each iteration. * n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. * n_epochs: Number of epochs. * input_height: This indicates the height of model's input. * input_width: This indicates the width of model's input. * weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved in a folder named "pretrained_model" in the same directory of "train.py" script. +* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved + in a folder named "pretrained_model" in the same directory of "train.py" script. * augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. * flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. * blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. @@ -255,9 +310,14 @@ The following parameter configuration can be applied to all segmentation use cas * brightness: The amount of brightenings. * thetha: Rotation angles. * degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. +* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the + training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the + models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from + model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". +* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train + and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we + write them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. * index_start: Starting index for saved models in the case that "continue_training" is ``true``. * dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. @@ -290,7 +350,8 @@ And the "dir_eval" the same structure as train directory: └── labels # directory of labels ``` -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following command, similar to the process for classification and reading order: +After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following +command, similar to the process for classification and reading order: `python train.py with config_classification.json` @@ -339,7 +400,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -384,7 +445,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -429,7 +490,7 @@ An example config json file for binarization can be like this: "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -439,7 +500,8 @@ An example config json file for binarization can be like this: } ``` -It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel image. +It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel +image. #### Page extraction @@ -476,7 +538,7 @@ It's important to mention that the value of n_classes for enhancement should be "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -486,7 +548,8 @@ It's important to mention that the value of n_classes for enhancement should be } ``` -For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, hence the patches parameter should be set to false. +For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, +hence the patches parameter should be set to false. #### layout segmentation @@ -533,7 +596,7 @@ An example config json file for layout segmentation with 5 classes (including ba "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, @@ -543,9 +606,11 @@ An example config json file for layout segmentation with 5 classes (including ba } ``` ## Inference with the trained model + ### classification -For conducting inference with a trained model, you simply need to execute the following command line, specifying the directory of the model and the image on which to perform inference: +For conducting inference with a trained model, you simply need to execute the following command line, specifying the +directory of the model and the image on which to perform inference: `python inference.py -m "model dir" -i "image" ` @@ -554,8 +619,9 @@ This will straightforwardly return the class of the image. ### machine based reading order - -To infer the reading order using an reading order model, we need a page XML file containing layout information but without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The new XML file with the added reading order will be written to the output directory with the same name. We need to run: +To infer the reading order using an reading order model, we need a page XML file containing layout information but +without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The +new XML file with the added reading order will be written to the output directory with the same name. We need to run: `python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` @@ -570,7 +636,8 @@ For conducting inference with a trained model for segmentation and enhancement y Note that in the case of page extraction the -p flag is not needed. -For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be calculated for the output. To do this, you need to provide the GT label using the argument -gt. +For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be +calculated for the output. To do this, you need to provide the GT label using the argument -gt. diff --git a/train/train.md b/train/train.md index 3eeb715..7e7ab63 100644 --- a/train/train.md +++ b/train/train.md @@ -4,7 +4,7 @@ This repository assists users in preparing training datasets, training models, a models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training dataset. -All these use cases are now utilized in the Eynollah workflow. +All these use cases are now utilized in the Eynollah workflow. As mentioned, the following three tasks can be accomplished using this repository: * Generate training dataset @@ -61,6 +61,7 @@ to the image area, with a default value of zero. To run the dataset generator, u pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. + To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four @@ -102,6 +103,7 @@ A possible custom config json file for layout segmentation where the "printspac "printspace_as_class_in_layout" : 8 } ``` + For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', @@ -199,7 +201,7 @@ For the classification use case, we haven't provided a ground truth generator, a The "dir_train" should be like this: -``` +``` . └── train # train directory ├── apple # directory of images for apple class @@ -208,7 +210,7 @@ The "dir_train" should be like this: And the "dir_eval" the same structure as train directory: -``` +``` . └── eval # evaluation directory ├── apple # directory of images for apple class @@ -277,11 +279,11 @@ classification and machine-based reading order, as you can see in their example * backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first -apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. + apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. * task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". * patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be -set to ``false``. + set to ``false``. * n_batch: Number of batches at each iteration. * n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. * n_epochs: Number of epochs. @@ -311,11 +313,11 @@ set to ``false``. * continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from - model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. + model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. * weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` * data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we - write them in sub-directories train and eval in "dir_output". + write them in sub-directories train and eval in "dir_output". * dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. * index_start: Starting index for saved models in the case that "continue_training" is ``true``. * dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. @@ -536,7 +538,7 @@ image. "thetha" : [10, -10], "continue_training": false, "index_start" : 0, - "dir_of_start_model" : " ", + "dir_of_start_model" : " ", "weighted_loss": false, "is_loss_soft_dice": false, "data_is_provided": false, From ce02a3553b084f9d30ade931a640e1d9711cf3e9 Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 15:18:21 +0200 Subject: [PATCH 247/374] :fire: remove obsolete versions of the training document --- docs/train_wiki.md | 643 --------------------------------------------- train/train.md | 643 --------------------------------------------- 2 files changed, 1286 deletions(-) delete mode 100644 docs/train_wiki.md delete mode 100644 train/train.md diff --git a/docs/train_wiki.md b/docs/train_wiki.md deleted file mode 100644 index 5158a80..0000000 --- a/docs/train_wiki.md +++ /dev/null @@ -1,643 +0,0 @@ -# Documentation - -This repository assists users in preparing training datasets, training models, and performing inference with trained -models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and -machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training -dataset. -All these use cases are now utilized in the Eynollah workflow. -As mentioned, the following three tasks can be accomplished using this repository: - -* Generate training dataset -* Train a model -* Inference with the trained model - -## Generate training dataset -The script generate_gt_for_training.py is used for generating training datasets. As the results of the following command demonstrate, the dataset generator provides three different commands: - -`python generate_gt_for_training.py --help` - - -These three commands are: - -* image-enhancement -* machine-based-reading-order -* pagexml2label - - -### image-enhancement - -Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of -high-resolution images. The training dataset can then be generated using the following command: - -`python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs "degrading scales json file"` - -The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are -downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose -resolution at different scales. The degraded images are used as input images, and the original high-resolution images -serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: - -```yaml -{ - "scales": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] -} -``` - -### machine-based-reading-order - -For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's -input is a three-channel image: the first and last channels contain information about each of the two text regions, -while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. -To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct -reading order. - -For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set -to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area -to the image area, with a default value of zero. To run the dataset generator, use the following command: - - -`python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` - -### pagexml2label - -pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, -including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. - -To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script -expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled -as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four -elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. - -In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired -element is automatically encoded as 1 in the PNG label. - -To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. -For example, in the case of 'textline' detection, the JSON file would resemble this: - -```yaml -{ -"use_case": "textline" -} -``` - -In the case of layout segmentation a possible custom config json file can be like this: - -```yaml -{ -"use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -} -``` - -A possible custom config json file for layout segmentation where the "printspace" is wished to be a class: - -```yaml -{ -"use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -"printspace_as_class_in_layout" : 8 -} -``` - -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a -given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an -image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', -and 'tableregion'. - -Text regions and graphic regions also have their own specific types. The known types for us for text regions are -'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', -'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', -'stamp', and 'signature'. - -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two -additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, -users can extract all known types from the labels and be confident that no unknown types are overlooked. - -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown -as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the -graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator -region" are also present in the label. However, other regions like "noise region" and "table region" will not be -included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. - -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" "` - -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key -is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, -the example JSON config file should look like this: - -```yaml -{ - "use_case": "layout", - "textregions": { - "paragraph": 1, - "drop-capital": 1, - "header": 2, - "heading": 2, - "marginalia": 3 - }, - "imageregion": 4, - "separatorregion": 5, - "graphicregions": { - "rest_as_decoration": 6 - }, - "artificial_class_on_boundary": ["paragraph", "header", "heading", "marginalia"], - "artificial_class_label": 7 -} -``` - -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the -elements labeled as "paragraph," "header," "heading," and "marginalia." - -For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the -"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements -represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the -artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use -case: - -```yaml -{ - "use_case": "textline", - "artificial_class_label": 2 -} -``` - -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to -crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that -in this scenario, since cropping will be applied to the label files, the directory of the original images must be -provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels -required for training are obtained. The command should resemble the following: - -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will be written" ` - -## Train a model -### classification - -For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, all we require is a training directory with subdirectories, each containing images of its respective classes. We need separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both directories. Additionally, the class names should be specified in the config JSON file, as shown in the following example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the "classification_classes_name" key in the config file should appear as follows: - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "classification", - "n_classes" : 2, - "n_epochs" : 10, - "input_height" : 448, - "input_width" : 448, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "f1_threshold_classification": 0.8, - "pretraining" : true, - "classification_classes_name" : {"0":"apple", "1":"orange"}, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── apple # directory of images for apple class - └── orange # directory of images for orange class -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── apple # directory of images for apple class - └── orange # directory of images for orange class - -``` - -The classification model can be trained using the following command line: - -`python train.py with config_classification.json` - - -As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. -This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, -an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". -Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". - -### reading order -An example config json file for machine based reading order should be like this: - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "reading_order", - "n_classes" : 1, - "n_epochs" : 5, - "input_height" : 672, - "input_width" : 448, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "pretraining" : true, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── images # directory of images - └── labels # directory of labels -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── images # directory of images - └── labels # directory of labels -``` - -The classification model can be trained like the classification case command line. - -### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement - -#### Parameter configuration for segmentation or enhancement usecases - -The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, -its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for -classification and machine-based reading order, as you can see in their example config files. - -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we - offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first - apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. -* task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this - parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be - set to ``false``. -* n_batch: Number of batches at each iteration. -* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. -* n_epochs: Number of epochs. -* input_height: This indicates the height of model's input. -* input_width: This indicates the width of model's input. -* weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved - in a folder named "pretrained_model" in the same directory of "train.py" script. -* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. -* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. -* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. -* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" parameter. -* degrading: If ``true``, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. -* brightening: If ``true``, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. -* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. -* rotation: If ``true``, 90 degree rotation will be applied on image. -* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. -* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. -* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. -* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. -* flip_index: Type of flips. -* blur_k: Type of blurrings. -* scales: Scales of scaling. -* brightness: The amount of brightenings. -* thetha: Rotation angles. -* degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the - training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the - models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from - model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. -* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train - and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we - write them in sub-directories train and eval in "dir_output". -* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. -* index_start: Starting index for saved models in the case that "continue_training" is ``true``. -* dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. -* transformer_num_patches_xy: Number of patches for vision transformer in x and y direction respectively. -* transformer_patchsize_x: Patch size of vision transformer patches in x direction. -* transformer_patchsize_y: Patch size of vision transformer patches in y direction. -* transformer_projection_dim: Transformer projection dimension. Default value is 64. -* transformer_mlp_head_units: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. -* transformer_layers: transformer layers. Default value is 8. -* transformer_num_heads: Transformer number of heads. Default value is 4. -* transformer_cnn_first: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. - -In the case of segmentation and enhancement the train and evaluation directory should be as following. - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── images # directory of images - └── labels # directory of labels -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── images # directory of images - └── labels # directory of labels -``` - -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following -command, similar to the process for classification and reading order: - -`python train.py with config_classification.json` - -#### Binarization - -An example config json file for binarization can be like this: - -```yaml -{ - "backbone_type" : "transformer", - "task": "binarization", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 224, - "input_width" : 672, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "transformer_num_patches_xy": [7, 7], - "transformer_patchsize_x": 3, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 192, - "transformer_mlp_head_units": [128, 64], - "transformer_layers": 8, - "transformer_num_heads": 4, - "transformer_cnn_first": true, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -#### Textline - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "segmentation", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -#### Enhancement - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "enhancement", - "n_classes" : 3, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel -image. - -#### Page extraction - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "segmentation", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : false, - "pretraining" : true, - "augmentation" : false, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, -hence the patches parameter should be set to false. - -#### layout segmentation - -An example config json file for layout segmentation with 5 classes (including background) can be like this: - -```yaml -{ - "backbone_type" : "transformer", - "task": "segmentation", - "n_classes" : 5, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "transformer_num_patches_xy": [7, 14], - "transformer_patchsize_x": 1, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 64, - "transformer_mlp_head_units": [128, 64], - "transformer_layers": 8, - "transformer_num_heads": 4, - "transformer_cnn_first": true, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` -## Inference with the trained model - -### classification - -For conducting inference with a trained model, you simply need to execute the following command line, specifying the -directory of the model and the image on which to perform inference: - - -`python inference.py -m "model dir" -i "image" ` - -This will straightforwardly return the class of the image. - -### machine based reading order - -To infer the reading order using an reading order model, we need a page XML file containing layout information but -without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The -new XML file with the added reading order will be written to the output directory with the same name. We need to run: - -`python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` - - -### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement - -For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: - - -`python inference.py -m "model dir" -i "image" -p -s "output image" ` - - -Note that in the case of page extraction the -p flag is not needed. - -For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be -calculated for the output. To do this, you need to provide the GT label using the argument -gt. - - - diff --git a/train/train.md b/train/train.md deleted file mode 100644 index 7e7ab63..0000000 --- a/train/train.md +++ /dev/null @@ -1,643 +0,0 @@ -# Documentation for Training Models - -This repository assists users in preparing training datasets, training models, and performing inference with trained -models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and -machine-based reading order. For each use case, we provide guidance on how to generate the corresponding training -dataset. -All these use cases are now utilized in the Eynollah workflow. -As mentioned, the following three tasks can be accomplished using this repository: - -* Generate training dataset -* Train a model -* Inference with the trained model - -## Generate training dataset -The script generate_gt_for_training.py is used for generating training datasets. As the results of the following command demonstrate, the dataset generator provides three different commands: - -`python generate_gt_for_training.py --help` - - -These three commands are: - -* image-enhancement -* machine-based-reading-order -* pagexml2label - - -### image-enhancement - -Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of -high-resolution images. The training dataset can then be generated using the following command: - -`python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs "degrading scales json file"` - -The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are -downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose -resolution at different scales. The degraded images are used as input images, and the original high-resolution images -serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: - -```yaml -{ - "scales": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] -} -``` - -### machine-based-reading-order - -For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's -input is a three-channel image: the first and last channels contain information about each of the two text regions, -while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. -To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct -reading order. - -For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set -to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area -to the image area, with a default value of zero. To run the dataset generator, use the following command: - - -`python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` - -### pagexml2label - -pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, -including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. - -To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script -expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled -as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four -elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. - -In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired -element is automatically encoded as 1 in the PNG label. - -To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. -For example, in the case of 'textline' detection, the JSON file would resemble this: - -```yaml -{ -"use_case": "textline" -} -``` - -In the case of layout segmentation a possible custom config json file can be like this: - -```yaml -{ -"use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -} -``` - -A possible custom config json file for layout segmentation where the "printspace" is wished to be a class: - -```yaml -{ -"use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -"printspace_as_class_in_layout" : 8 -} -``` - -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. In a -given image, the annotations of elements are recorded in a page XML file, including their contours and classes. For an -image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', 'noiseregion', -and 'tableregion'. - -Text regions and graphic regions also have their own specific types. The known types for us for text regions are -'paragraph', 'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', -'page-number', and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', -'stamp', and 'signature'. - -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined two -additional types: "rest_as_paragraph" and "rest_as_decoration" to ensure that no unknown types are missed. This way, -users can extract all known types from the labels and be confident that no unknown types are overlooked. - -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown -as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the -graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator -region" are also present in the label. However, other regions like "noise region" and "table region" will not be -included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. - -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" "` - -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key -is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, -the example JSON config file should look like this: - -```yaml -{ - "use_case": "layout", - "textregions": { - "paragraph": 1, - "drop-capital": 1, - "header": 2, - "heading": 2, - "marginalia": 3 - }, - "imageregion": 4, - "separatorregion": 5, - "graphicregions": { - "rest_as_decoration": 6 - }, - "artificial_class_on_boundary": ["paragraph", "header", "heading", "marginalia"], - "artificial_class_label": 7 -} -``` - -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the -elements labeled as "paragraph," "header," "heading," and "marginalia." - -For "textline," "word," and "glyph," the artificial class on the boundaries will be activated only if the -"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements -represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the -artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use -case: - -```yaml -{ - "use_case": "textline", - "artificial_class_label": 2 -} -``` - -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to -crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that -in this scenario, since cropping will be applied to the label files, the directory of the original images must be -provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels -required for training are obtained. The command should resemble the following: - -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will be written" ` - -## Train a model -### classification - -For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, all we require is a training directory with subdirectories, each containing images of its respective classes. We need separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both directories. Additionally, the class names should be specified in the config JSON file, as shown in the following example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the "classification_classes_name" key in the config file should appear as follows: - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "classification", - "n_classes" : 2, - "n_epochs" : 10, - "input_height" : 448, - "input_width" : 448, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "f1_threshold_classification": 0.8, - "pretraining" : true, - "classification_classes_name" : {"0":"apple", "1":"orange"}, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── apple # directory of images for apple class - └── orange # directory of images for orange class -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── apple # directory of images for apple class - └── orange # directory of images for orange class - -``` - -The classification model can be trained using the following command line: - -`python train.py with config_classification.json` - - -As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. -This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, -an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". -Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". - -### reading order -An example config json file for machine based reading order should be like this: - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "reading_order", - "n_classes" : 1, - "n_epochs" : 5, - "input_height" : 672, - "input_width" : 448, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "pretraining" : true, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── images # directory of images - └── labels # directory of labels -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── images # directory of images - └── labels # directory of labels -``` - -The classification model can be trained like the classification case command line. - -### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement - -#### Parameter configuration for segmentation or enhancement usecases - -The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, -its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for -classification and machine-based reading order, as you can see in their example config files. - -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we - offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first - apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. -* task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this - parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be - set to ``false``. -* n_batch: Number of batches at each iteration. -* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it should set to 1. And for the case of layout detection just the unique number of classes should be given. -* n_epochs: Number of epochs. -* input_height: This indicates the height of model's input. -* input_width: This indicates the width of model's input. -* weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved - in a folder named "pretrained_model" in the same directory of "train.py" script. -* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. -* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. -* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. -* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" parameter. -* degrading: If ``true``, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. -* brightening: If ``true``, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. -* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. -* rotation: If ``true``, 90 degree rotation will be applied on image. -* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. -* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. -* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. -* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. -* flip_index: Type of flips. -* blur_k: Type of blurrings. -* scales: Scales of scaling. -* brightness: The amount of brightenings. -* thetha: Rotation angles. -* degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the - training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the - models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from - model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. -* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train - and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we - write them in sub-directories train and eval in "dir_output". -* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. -* index_start: Starting index for saved models in the case that "continue_training" is ``true``. -* dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. -* transformer_num_patches_xy: Number of patches for vision transformer in x and y direction respectively. -* transformer_patchsize_x: Patch size of vision transformer patches in x direction. -* transformer_patchsize_y: Patch size of vision transformer patches in y direction. -* transformer_projection_dim: Transformer projection dimension. Default value is 64. -* transformer_mlp_head_units: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. -* transformer_layers: transformer layers. Default value is 8. -* transformer_num_heads: Transformer number of heads. Default value is 4. -* transformer_cnn_first: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. - -In the case of segmentation and enhancement the train and evaluation directory should be as following. - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── images # directory of images - └── labels # directory of labels -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── images # directory of images - └── labels # directory of labels -``` - -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following -command, similar to the process for classification and reading order: - -`python train.py with config_classification.json` - -#### Binarization - -An example config json file for binarization can be like this: - -```yaml -{ - "backbone_type" : "transformer", - "task": "binarization", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 224, - "input_width" : 672, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "transformer_num_patches_xy": [7, 7], - "transformer_patchsize_x": 3, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 192, - "transformer_mlp_head_units": [128, 64], - "transformer_layers": 8, - "transformer_num_heads": 4, - "transformer_cnn_first": true, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -#### Textline - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "segmentation", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -#### Enhancement - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "enhancement", - "n_classes" : 3, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel -image. - -#### Page extraction - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "segmentation", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : false, - "pretraining" : true, - "augmentation" : false, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -For page segmentation (or print space or border segmentation), the model needs to view the input image in its entirety, -hence the patches parameter should be set to false. - -#### layout segmentation - -An example config json file for layout segmentation with 5 classes (including background) can be like this: - -```yaml -{ - "backbone_type" : "transformer", - "task": "segmentation", - "n_classes" : 5, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "transformer_num_patches_xy": [7, 14], - "transformer_patchsize_x": 1, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 64, - "transformer_mlp_head_units": [128, 64], - "transformer_layers": 8, - "transformer_num_heads": 4, - "transformer_cnn_first": true, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` -## Inference with the trained model - -### classification - -For conducting inference with a trained model, you simply need to execute the following command line, specifying the -directory of the model and the image on which to perform inference: - - -`python inference.py -m "model dir" -i "image" ` - -This will straightforwardly return the class of the image. - -### machine based reading order - -To infer the reading order using an reading order model, we need a page XML file containing layout information but -without the reading order. We simply need to provide the model directory, the XML file, and the output directory. The -new XML file with the added reading order will be written to the output directory with the same name. We need to run: - -`python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` - - -### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement - -For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: - - -`python inference.py -m "model dir" -i "image" -p -s "output image" ` - - -Note that in the case of page extraction the -p flag is not needed. - -For segmentation or binarization tasks, if a ground truth (GT) label is available, the IOU evaluation metric can be -calculated for the output. To do this, you need to provide the GT label using the argument -gt. - - - From 2bcd20ebc740ba17fd1af11910cb9a2983da68e6 Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 15:21:42 +0200 Subject: [PATCH 248/374] reference the now-merged training tools in README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4683eb7..e576f4d 100644 --- a/README.md +++ b/README.md @@ -53,13 +53,16 @@ make install EXTRAS=OCR ``` ## Models + Pretrained models can be downloaded from [zenodo](https://zenodo.org/records/17194824) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). -## Train +## Training -In case you want to train your own model with Eynollah, have a look at [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md). +In case you want to train your own model with Eynollah, have see the +documentation in [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md) and use the +tools in the [`train` folder](https://github.com/qurator-spk/eynollah/tree/main/train). ## Usage From 9d8b858dfc9099f25c928adf39d4096309ced200 Mon Sep 17 00:00:00 2001 From: kba Date: Mon, 29 Sep 2025 16:01:29 +0200 Subject: [PATCH 249/374] remove docs/eynollah-layout, superseded by docs/model.md and docs/usage.md --- .gitignore | 1 + docs/eynollah-layout.md | 100 ---------------------------------------- 2 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 docs/eynollah-layout.md diff --git a/.gitignore b/.gitignore index 0d5d834..da03449 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ output.html /build /dist *.tif +*.sw? diff --git a/docs/eynollah-layout.md b/docs/eynollah-layout.md deleted file mode 100644 index e76ed51..0000000 --- a/docs/eynollah-layout.md +++ /dev/null @@ -1,100 +0,0 @@ -# `eynollah layout` documentation - -Eynollah can currently be used to detect the following region types/elements: -* Background -* [Border](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_BorderType.html) -* [Textregion](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextRegionType.html) -* [Textline](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextLineType.html) -* [Header](https://ocr-d.de/en/gt-guidelines/trans/lyUeberschrift.html) -* [Image](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_ImageRegionType.html) -* [Separator](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_SeparatorRegionType.html) -* [Marginalia](https://ocr-d.de/en/gt-guidelines/trans/lyMarginalie.html) -* [Initial (Drop Capital)](https://ocr-d.de/en/gt-guidelines/trans/lyInitiale.html) -* [Table](https://ocr-d.de/en/gt-guidelines/trans/lyTabellen.html) - -In addition, the tool can detect the [ReadingOrder](https://ocr-d.de/en/gt-guidelines/trans/lyLeserichtung.html) of text regions, both from left-to-right or from right-to-left. The final goal is to feed the output to an OCR model. - -## Method description - -Eynollah is based on pixelwise segmentation using a combination of a ResNet50 encoder with various U-Net decoders. -It uses a combination of multiple models and heuristics (see the flowchart below for the different stages and how they interact): -* [Border detection](https://github.com/qurator-spk/eynollah#border-detection) -* [Layout detection](https://github.com/qurator-spk/eynollah#layout-detection) -* [Textline detection](https://github.com/qurator-spk/eynollah#textline-detection) -* [Image enhancement](https://github.com/qurator-spk/eynollah#Image_enhancement) -* [Scale classification](https://github.com/qurator-spk/eynollah#Scale_classification) -* [Heuristic methods](https://https://github.com/qurator-spk/eynollah#heuristic-methods) - -![](https://user-images.githubusercontent.com/952378/100619946-1936f680-331e-11eb-9297-6e8b4cab3c16.png) - -### Border detection -For the purpose of text recognition (OCR) and in order to avoid noise being introduced from texts outside the printspace, one first needs to detect the border of the printed frame. This is done by a binary pixel-wise-segmentation model trained on a dataset of 2,000 documents where about 1,200 of them come from the [dhSegment](https://github.com/dhlab-epfl/dhSegment/) project (you can download the dataset from [here](https://github.com/dhlab-epfl/dhSegment/releases/download/v0.2/pages.zip)) and the remainder having been annotated in SBB. For border detection, the model needs to be fed with the whole image at once rather than separated in patches. - -### Layout detection -As a next step, text regions need to be identified by means of layout detection. Again a pixel-wise segmentation model was trained on 131 labeled images from the SBB digital collections, including some data augmentation. Since the target of this tool are historical documents, we consider as main region types text regions, separators, images, tables and background - each with their own subclasses, e.g. in the case of text regions, subclasses like header/heading, drop capital, main body text etc. While it would be desirable to detect and classify each of these classes in a granular way, there are also limitations due to having a suitably large and balanced training set. Accordingly, the current version of this tool is focussed on the main region types background, text region, image and separator. - -### Textline detection -In a subsequent step, binary pixel-wise segmentation is used again to classify pixels in a document that constitute textlines. For textline segmentation, a model was initially trained on documents with only one column/block of text and some augmentation with regard to scaling. By fine-tuning the parameters also for multi-column documents, additional training data was produced that resulted in a much more robust textline detection model. - -### Image enhancement -This is an image to image model which input was low quality of an image and label was actually the original image. For this one we did not have any GT, so we decreased the quality of documents in SBB and then feed them into model. - -### Scale classification -This is simply an image classifier which classifies images based on their scales or better to say based on their number of columns. - -### Heuristic methods -Some heuristic methods are also employed to further improve the model predictions: -* After border detection, the largest contour is determined by a bounding box, and the image cropped to these coordinates. -* For text region detection, the image is scaled up to make it easier for the model to detect background space between text regions. -* A minimum area is defined for text regions in relation to the overall image dimensions, so that very small regions that are noise can be filtered out. -* Deskewing is applied on the text region level (due to regions having different degrees of skew) in order to improve the textline segmentation result. -* After deskewing, a calculation of the pixel distribution on the X-axis allows the separation of textlines (foreground) and background pixels. -* Finally, using the derived coordinates, bounding boxes are determined for each textline. - -## Models - -TODO - -## How to use - -First, this model makes use of up to 9 trained models which are responsible for different operations like size detection, column classification, image enhancement, page extraction, main layout detection, full layout detection and textline detection.That does not mean that all 9 models are always required for every document. Based on the document characteristics and parameters specified, different scenarios can be applied. - -* If none of the parameters is set to `true`, the tool will perform a layout detection of main regions (background, text, images, separators and marginals). An advantage of this tool is that it tries to extract main text regions separately as much as possible. - -* If you set `-ae` (**a**llow image **e**nhancement) parameter to `true`, the tool will first check the ppi (pixel-per-inch) of the image and when it is less than 300, the tool will resize it and only then image enhancement will occur. Image enhancement can also take place without this option, but by setting this option to `true`, the layout xml data (e.g. coordinates) will be based on the resized and enhanced image instead of the original image. - -* For some documents, while the quality is good, their scale is very large, and the performance of tool decreases. In such cases you can set `-as` (**a**llow **s**caling) to `true`. With this option enabled, the tool will try to rescale the image and only then the layout detection process will begin. - -* If you care about drop capitals (initials) and headings, you can set `-fl` (**f**ull **l**ayout) to `true`. With this setting, the tool can currently distinguish 7 document layout classes/elements. - -* In cases where the document includes curved headers or curved lines, rectangular bounding boxes for textlines will not be a great option. In such cases it is strongly recommended setting the flag `-cl` (**c**urved **l**ines) to `true` to find contours of curved lines instead of rectangular bounding boxes. Be advised that enabling this option increases the processing time of the tool. - -* To crop and save image regions inside the document, set the parameter `-si` (**s**ave **i**mages) to true and provide a directory path to store the extracted images. - -* To extract only images from a document, set the parameter `-eoi` (**e**xtract **o**nly **i**mages). Choosing this option disables any other processing. To save the cropped images add `-ep` and `-si`. - -* This tool is actively being developed. If problems occur, or the performance does not meet your expectations, we welcome your feedback via [issues](https://github.com/qurator-spk/eynollah/issues). - - -### `--full-layout` vs `--no-full-layout` - -Here are the difference in elements detected depending on the `--full-layout`/`--no-full-layout` command line flags: - -| | `--full-layout` | `--no-full-layout` | -| --- | --- | --- | -| reading order | x | x | -| header regions | x | - | -| text regions | x | x | -| text regions / text line | x | x | -| drop-capitals | x | - | -| marginals | x | x | -| marginals / text line | x | x | -| image region | x | x | - -### Use as OCR-D processor - -Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) processor. In this case, the source image file group with (preferably) RGB images should be used as input (the image provided by `@imageFilename` is passed on directly): - -`ocrd-eynollah-segment -I OCR-D-IMG -O SEG-LINE -P models` - -## Examples From 09ece86f0dcb860eef978319b2350ccf7df13c2c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 19 Aug 2025 11:58:45 +0200 Subject: [PATCH 250/374] dilate_textregions_contours: simplify (via shapely's Polygon.buffer()), ensure validity --- src/eynollah/eynollah.py | 212 ++-------------------------------- src/eynollah/utils/contour.py | 30 ++++- 2 files changed, 36 insertions(+), 206 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index d47016b..55789ae 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -27,6 +27,7 @@ from loky import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 import numpy as np +from shapely.geometry import Polygon from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d from numba import cuda @@ -68,6 +69,7 @@ from .utils.contour import ( get_text_region_boxes_by_given_contours, get_textregion_contours_in_org_image, get_textregion_contours_in_org_image_light, + make_valid, return_contours_of_image, return_contours_of_interested_region, return_contours_of_interested_region_by_min_size, @@ -3670,211 +3672,15 @@ class Eynollah: return x_differential_new def dilate_textregions_contours_textline_version(self, all_found_textline_polygons): - #print(all_found_textline_polygons) - for j in range(len(all_found_textline_polygons)): - for ij in range(len(all_found_textline_polygons[j])): - con_ind = all_found_textline_polygons[j][ij] - area = cv2.contourArea(con_ind) - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_differential = gaussian_filter1d(x_differential, 0.1) - y_differential = gaussian_filter1d(y_differential, 0.1) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - x_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in x_differential] - y_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in y_differential] - - abs_diff=abs(abs(x_differential)- abs(y_differential) ) - - inc_x = np.zeros(len(x_differential)+1) - inc_y = np.zeros(len(x_differential)+1) - - if (y_max-y_min) <= (x_max-x_min): - dilation_m1 = round(area / (x_max-x_min) * 0.12) - else: - dilation_m1 = round(area / (y_max-y_min) * 0.12) - - if dilation_m1>8: - dilation_m1 = 8 - if dilation_m1<6: - dilation_m1 = 6 - #print(dilation_m1, 'dilation_m1') - dilation_m1 = 6 - dilation_m2 = int(dilation_m1/2.) +1 - - for i in range(len(x_differential)): - if abs_diff[i]==0: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]==0 and y_differential_mask_nonzeros[i]!=0: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]!=0 and y_differential_mask_nonzeros[i]==0: - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - - elif abs_diff[i]!=0 and abs_diff[i]>=3: - if abs(x_differential[i])>abs(y_differential[i]): - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - else: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - else: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - - inc_x[0] = inc_x[-1] - inc_y[0] = inc_y[-1] - - con_scaled = con_ind*1 - - con_scaled[:,0, 0] = con_ind[:,0,0] + np.array(inc_x)[:] - con_scaled[:,0, 1] = con_ind[:,0,1] + np.array(inc_y)[:] - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - area_scaled = cv2.contourArea(con_scaled.astype(np.int32)) - - con_ind = con_ind.astype(np.int32) - - results = [cv2.pointPolygonTest(con_ind, (con_scaled[ind,0, 0], con_scaled[ind,0, 1]), False) - for ind in range(len(con_scaled[:,0, 1])) ] - results = np.array(results) - #print(results,'results') - results[results==0] = 1 - - diff_result = np.diff(results) - - indices_2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==2] - indices_m2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==-2] - - if results[0]==1: - con_scaled[:indices_m2[0]+1,0, 1] = con_ind[:indices_m2[0]+1,0,1] - con_scaled[:indices_m2[0]+1,0, 0] = con_ind[:indices_m2[0]+1,0,0] - #indices_2 = indices_2[1:] - indices_m2 = indices_m2[1:] - - if len(indices_2)>len(indices_m2): - con_scaled[indices_2[-1]+1:,0, 1] = con_ind[indices_2[-1]+1:,0,1] - con_scaled[indices_2[-1]+1:,0, 0] = con_ind[indices_2[-1]+1:,0,0] - indices_2 = indices_2[:-1] - - for ii in range(len(indices_2)): - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 1] = con_scaled[indices_2[ii],0, 1] - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 0] = con_scaled[indices_2[ii],0, 0] - - all_found_textline_polygons[j][ij][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][ij][:,0,0] = con_scaled[:,0, 0] - return all_found_textline_polygons + return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords, + dtype=int)[:, np.newaxis] + for poly in region] + for region in all_found_textline_polygons] def dilate_textregions_contours(self, all_found_textline_polygons): - #print(all_found_textline_polygons) - for j in range(len(all_found_textline_polygons)): - con_ind = all_found_textline_polygons[j] - #print(len(con_ind[:,0,0]),'con_ind[:,0,0]') - area = cv2.contourArea(con_ind) - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_differential = gaussian_filter1d(x_differential, 0.1) - y_differential = gaussian_filter1d(y_differential, 0.1) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - x_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in x_differential] - y_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in y_differential] - - abs_diff=abs(abs(x_differential)- abs(y_differential) ) - - inc_x = np.zeros(len(x_differential)+1) - inc_y = np.zeros(len(x_differential)+1) - - if (y_max-y_min) <= (x_max-x_min): - dilation_m1 = round(area / (x_max-x_min) * 0.12) - else: - dilation_m1 = round(area / (y_max-y_min) * 0.12) - - if dilation_m1>8: - dilation_m1 = 8 - if dilation_m1<6: - dilation_m1 = 6 - #print(dilation_m1, 'dilation_m1') - dilation_m1 = 6 - dilation_m2 = int(dilation_m1/2.) +1 - - for i in range(len(x_differential)): - if abs_diff[i]==0: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]==0 and y_differential_mask_nonzeros[i]!=0: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]!=0 and y_differential_mask_nonzeros[i]==0: - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - - elif abs_diff[i]!=0 and abs_diff[i]>=3: - if abs(x_differential[i])>abs(y_differential[i]): - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - else: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - else: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - - inc_x[0] = inc_x[-1] - inc_y[0] = inc_y[-1] - - con_scaled = con_ind*1 - - con_scaled[:,0, 0] = con_ind[:,0,0] + np.array(inc_x)[:] - con_scaled[:,0, 1] = con_ind[:,0,1] + np.array(inc_y)[:] - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - area_scaled = cv2.contourArea(con_scaled.astype(np.int32)) - - con_ind = con_ind.astype(np.int32) - - results = [cv2.pointPolygonTest(con_ind, (con_scaled[ind,0, 0], con_scaled[ind,0, 1]), False) - for ind in range(len(con_scaled[:,0, 1])) ] - results = np.array(results) - #print(results,'results') - results[results==0] = 1 - - diff_result = np.diff(results) - indices_2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==2] - indices_m2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==-2] - - if results[0]==1: - con_scaled[:indices_m2[0]+1,0, 1] = con_ind[:indices_m2[0]+1,0,1] - con_scaled[:indices_m2[0]+1,0, 0] = con_ind[:indices_m2[0]+1,0,0] - #indices_2 = indices_2[1:] - indices_m2 = indices_m2[1:] - - if len(indices_2)>len(indices_m2): - con_scaled[indices_2[-1]+1:,0, 1] = con_ind[indices_2[-1]+1:,0,1] - con_scaled[indices_2[-1]+1:,0, 0] = con_ind[indices_2[-1]+1:,0,0] - indices_2 = indices_2[:-1] - - for ii in range(len(indices_2)): - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 1] = con_scaled[indices_2[ii],0, 1] - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 0] = con_scaled[indices_2[ii],0, 0] - - all_found_textline_polygons[j][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][:,0,0] = con_scaled[:,0, 0] - return all_found_textline_polygons + return [np.array(make_valid(Polygon(poly[:, 0])).buffer(5).exterior.coords, + dtype=int)[:, np.newaxis] + for poly in all_found_textline_polygons] def dilate_textline_contours(self, all_found_textline_polygons): for j in range(len(all_found_textline_polygons)): diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 0e84153..3d7e5c8 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -1,7 +1,7 @@ from functools import partial import cv2 import numpy as np -from shapely import geometry +from shapely.geometry import Polygon from .rotate import rotate_image, rotation_image_new @@ -43,7 +43,7 @@ def filter_contours_area_of_image(image, contours, hierarchy, max_area, min_area if len(c) < 3: # A polygon cannot have less than 3 points continue - polygon = geometry.Polygon([point[0] for point in c]) + polygon = Polygon([point[0] for point in c]) area = polygon.area if (area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and @@ -58,7 +58,7 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m if len(c) < 3: # A polygon cannot have less than 3 points continue - polygon = geometry.Polygon([point[0] for point in c]) + polygon = Polygon([point[0] for point in c]) # area = cv2.contourArea(c) area = polygon.area ##print(np.prod(thresh.shape[:2])) @@ -332,3 +332,27 @@ def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, return img_ret[:, :, 0] +def make_valid(polygon: Polygon) -> Polygon: + """Ensures shapely.geometry.Polygon object is valid by repeated rearrangement/simplification/enlargement.""" + points = list(polygon.exterior.coords) + # try by re-arranging points + for split in range(1, len(points)): + if polygon.is_valid or polygon.simplify(polygon.area).is_valid: + break + # simplification may not be possible (at all) due to ordering + # in that case, try another starting point + polygon = Polygon(points[-split:]+points[:-split]) + # try by simplification + for tolerance in range(int(polygon.area + 1.5)): + if polygon.is_valid: + break + # simplification may require a larger tolerance + polygon = polygon.simplify(tolerance + 1) + # try by enlarging + for tolerance in range(1, int(polygon.area + 2.5)): + if polygon.is_valid: + break + # enlargement may require a larger tolerance + polygon = polygon.buffer(tolerance) + assert polygon.is_valid, polygon.wkt + return polygon From b48c41e68ff59d8cff97a59a534fee20d2d32408 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 19 Aug 2025 20:09:09 +0200 Subject: [PATCH 251/374] return_boxes_of_images_by_order_of_reading_new: simplify, avoid changing dtype during np.append --- src/eynollah/eynollah.py | 2 +- src/eynollah/utils/__init__.py | 214 +++++++++++++++------------------ 2 files changed, 97 insertions(+), 119 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 55789ae..959e9a6 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3678,7 +3678,7 @@ class Eynollah: for region in all_found_textline_polygons] def dilate_textregions_contours(self, all_found_textline_polygons): - return [np.array(make_valid(Polygon(poly[:, 0])).buffer(5).exterior.coords, + return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords, dtype=int)[:, np.newaxis] for poly in all_found_textline_polygons] diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index c5962f8..7168d95 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1632,6 +1632,7 @@ def return_boxes_of_images_by_order_of_reading_new( regions_without_separators = cv2.flip(regions_without_separators,1) boxes=[] peaks_neg_tot_tables = [] + splitter_y_new = np.array(splitter_y_new, dtype=int) for i in range(len(splitter_y_new)-1): #print(splitter_y_new[i],splitter_y_new[i+1]) matrix_new = matrix_of_lines_ch[:,:][(matrix_of_lines_ch[:,6]> splitter_y_new[i] ) & @@ -1644,14 +1645,9 @@ def return_boxes_of_images_by_order_of_reading_new( # 0.1 * (np.abs(splitter_y_new[i+1]-splitter_y_new[i]))): if True: try: - if erosion_hurts: - num_col, peaks_neg_fin = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], - num_col_classifier, tables, multiplier=6.) - else: - num_col, peaks_neg_fin = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], - num_col_classifier, tables, multiplier=7.) + num_col, peaks_neg_fin = find_num_col( + regions_without_separators[splitter_y_new[i]:splitter_y_new[i+1], :], + num_col_classifier, tables, multiplier=6. if erosion_hurts else 7.) except: peaks_neg_fin=[] num_col = 0 @@ -1661,7 +1657,7 @@ def return_boxes_of_images_by_order_of_reading_new( #print('burda') if len(peaks_neg_fin)==0: num_col, peaks_neg_fin = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], + regions_without_separators[splitter_y_new[i]:splitter_y_new[i+1], :], num_col_classifier, tables, multiplier=3.) peaks_neg_fin_early=[] peaks_neg_fin_early.append(0) @@ -1674,21 +1670,21 @@ def return_boxes_of_images_by_order_of_reading_new( peaks_neg_fin_rev=[] for i_n in range(len(peaks_neg_fin_early)-1): #print(i_n,'i_n') - #plt.plot(regions_without_separators[int(splitter_y_new[i]): - # int(splitter_y_new[i+1]), + #plt.plot(regions_without_separators[splitter_y_new[i]: + # splitter_y_new[i+1], # peaks_neg_fin_early[i_n]: # peaks_neg_fin_early[i_n+1]].sum(axis=0) ) #plt.show() try: num_col, peaks_neg_fin1 = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]), + regions_without_separators[splitter_y_new[i]:splitter_y_new[i+1], peaks_neg_fin_early[i_n]:peaks_neg_fin_early[i_n+1]], num_col_classifier,tables, multiplier=7.) except: peaks_neg_fin1=[] try: num_col, peaks_neg_fin2 = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]), + regions_without_separators[splitter_y_new[i]:splitter_y_new[i+1], peaks_neg_fin_early[i_n]:peaks_neg_fin_early[i_n+1]], num_col_classifier,tables, multiplier=5.) except: @@ -1716,7 +1712,7 @@ def return_boxes_of_images_by_order_of_reading_new( except: pass #num_col, peaks_neg_fin = find_num_col( - # regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], + # regions_without_separators[splitter_y_new[i]:splitter_y_new[i+1],:], # multiplier=7.0) x_min_hor_some=matrix_new[:,2][ (matrix_new[:,9]==0) ] x_max_hor_some=matrix_new[:,3][ (matrix_new[:,9]==0) ] @@ -1738,31 +1734,28 @@ def return_boxes_of_images_by_order_of_reading_new( y_lines_with_child_without_mother, x_start_with_child_without_mother, x_end_with_child_without_mother, \ new_main_sep_y = return_x_start_end_mothers_childs_and_type_of_reading_order( x_min_hor_some, x_max_hor_some, cy_hor_some, peaks_neg_tot, cy_hor_diff) - x_starting = np.array(x_starting) - x_ending = np.array(x_ending) - y_type_2 = np.array(y_type_2) - y_diff_type_2 = np.array(y_diff_type_2) + all_columns = set(range(len(peaks_neg_tot) - 1)) if ((reading_order_type==1) or (reading_order_type==0 and (len(y_lines_without_mother)>=2 or there_is_sep_with_child==1))): try: - y_grenze=int(splitter_y_new[i])+300 + y_grenze = splitter_y_new[i] + 300 #check if there is a big separator in this y_mains_sep_ohne_grenzen args_early_ys=np.arange(len(y_type_2)) #print(args_early_ys,'args_early_ys') - #print(int(splitter_y_new[i]),int(splitter_y_new[i+1])) + #print(splitter_y_new[i], splitter_y_new[i+1]) - x_starting_up = x_starting[(y_type_2 > int(splitter_y_new[i])) & + x_starting_up = x_starting[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - x_ending_up = x_ending[(y_type_2 > int(splitter_y_new[i])) & + x_ending_up = x_ending[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - y_type_2_up = y_type_2[(y_type_2 > int(splitter_y_new[i])) & + y_type_2_up = y_type_2[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - y_diff_type_2_up = y_diff_type_2[(y_type_2 > int(splitter_y_new[i])) & + y_diff_type_2_up = y_diff_type_2[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - args_up = args_early_ys[(y_type_2 > int(splitter_y_new[i])) & + args_up = args_early_ys[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] if len(y_type_2_up) > 0: y_main_separator_up = y_type_2_up [(x_starting_up==0) & @@ -1776,8 +1769,8 @@ def return_boxes_of_images_by_order_of_reading_new( args_to_be_kept = np.array(list( set(args_early_ys) - set(args_main_to_deleted) )) #print(args_to_be_kept,'args_to_be_kept') boxes.append([0, peaks_neg_tot[len(peaks_neg_tot)-1], - int(splitter_y_new[i]), int( np.max(y_diff_main_separator_up))]) - splitter_y_new[i]=[ np.max(y_diff_main_separator_up) ][0] + splitter_y_new[i], y_diff_main_separator_up.max()]) + splitter_y_new[i] = y_diff_main_separator_up.max() #print(splitter_y_new[i],'splitter_y_new[i]') y_type_2 = y_type_2[args_to_be_kept] @@ -1786,29 +1779,28 @@ def return_boxes_of_images_by_order_of_reading_new( y_diff_type_2 = y_diff_type_2[args_to_be_kept] #print('galdiha') - y_grenze=int(splitter_y_new[i])+200 + y_grenze = splitter_y_new[i] + 200 args_early_ys2=np.arange(len(y_type_2)) - y_type_2_up=y_type_2[(y_type_2 > int(splitter_y_new[i])) & + y_type_2_up=y_type_2[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - x_starting_up=x_starting[(y_type_2 > int(splitter_y_new[i])) & + x_starting_up=x_starting[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - x_ending_up=x_ending[(y_type_2 > int(splitter_y_new[i])) & + x_ending_up=x_ending[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - y_diff_type_2_up=y_diff_type_2[(y_type_2 > int(splitter_y_new[i])) & + y_diff_type_2_up=y_diff_type_2[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] - args_up2=args_early_ys2[(y_type_2 > int(splitter_y_new[i])) & + args_up2=args_early_ys2[(y_type_2 > splitter_y_new[i]) & (y_type_2 <= y_grenze)] #print(y_type_2_up,x_starting_up,x_ending_up,'didid') - nodes_in = [] + nodes_in = set() for ij in range(len(x_starting_up)): - nodes_in = nodes_in + list(range(x_starting_up[ij], - x_ending_up[ij])) - nodes_in = np.unique(nodes_in) + nodes_in.update(range(x_starting_up[ij], + x_ending_up[ij])) #print(nodes_in,'nodes_in') - if set(nodes_in)==set(range(len(peaks_neg_tot)-1)): + if nodes_in == set(range(len(peaks_neg_tot)-1)): pass - elif set(nodes_in)==set(range(1, len(peaks_neg_tot)-1)): + elif nodes_in == set(range(1, len(peaks_neg_tot)-1)): pass else: #print('burdaydikh') @@ -1823,17 +1815,16 @@ def return_boxes_of_images_by_order_of_reading_new( pass #print('burdaydikh2') elif len(y_diff_main_separator_up)==0: - nodes_in = [] + nodes_in = set() for ij in range(len(x_starting_up)): - nodes_in = nodes_in + list(range(x_starting_up[ij], - x_ending_up[ij])) - nodes_in = np.unique(nodes_in) + nodes_in.update(range(x_starting_up[ij], + x_ending_up[ij])) #print(nodes_in,'nodes_in2') #print(np.array(range(len(peaks_neg_tot)-1)),'np.array(range(len(peaks_neg_tot)-1))') - if set(nodes_in)==set(range(len(peaks_neg_tot)-1)): + if nodes_in == set(range(len(peaks_neg_tot)-1)): pass - elif set(nodes_in)==set(range(1,len(peaks_neg_tot)-1)): + elif nodes_in == set(range(1,len(peaks_neg_tot)-1)): pass else: #print('burdaydikh') @@ -1858,26 +1849,24 @@ def return_boxes_of_images_by_order_of_reading_new( x_end_by_order=[] if (len(x_end_with_child_without_mother)==0 and reading_order_type==0) or reading_order_type==1: if reading_order_type==1: - y_lines_by_order.append(int(splitter_y_new[i])) + y_lines_by_order.append(splitter_y_new[i]) x_start_by_order.append(0) x_end_by_order.append(len(peaks_neg_tot)-2) else: #print(x_start_without_mother,x_end_without_mother,peaks_neg_tot,'dodo') - columns_covered_by_mothers = [] + columns_covered_by_mothers = set() for dj in range(len(x_start_without_mother)): - columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_start_without_mother[dj], - x_end_without_mother[dj])) - columns_covered_by_mothers = list(set(columns_covered_by_mothers)) - - all_columns=np.arange(len(peaks_neg_tot)-1) - columns_not_covered=list(set(all_columns) - set(columns_covered_by_mothers)) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * (len(columns_not_covered) + len(x_start_without_mother))) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) + columns_covered_by_mothers.update( + range(x_start_without_mother[dj], + x_end_without_mother[dj])) + columns_not_covered = list(all_columns - columns_covered_by_mothers) + y_type_2 = np.append(y_type_2, np.ones(len(columns_not_covered) + len(x_start_without_mother), + dtype=int) * splitter_y_new[i]) + ##y_lines_by_order = np.append(y_lines_by_order, [splitter_y_new[i]] * len(columns_not_covered)) ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) + x_starting = np.append(x_starting, np.array(columns_not_covered, int)) x_starting = np.append(x_starting, x_start_without_mother) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) + x_ending = np.append(x_ending, np.array(columns_not_covered, int) + 1) x_ending = np.append(x_ending, x_end_without_mother) ind_args=np.arange(len(y_type_2)) @@ -1906,39 +1895,34 @@ def return_boxes_of_images_by_order_of_reading_new( x_end_by_order.append(x_end_column_sort[ii]-1) else: #print(x_start_without_mother,x_end_without_mother,peaks_neg_tot,'dodo') - columns_covered_by_mothers = [] + columns_covered_by_mothers = set() for dj in range(len(x_start_without_mother)): - columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_start_without_mother[dj], - x_end_without_mother[dj])) - columns_covered_by_mothers = list(set(columns_covered_by_mothers)) - - all_columns=np.arange(len(peaks_neg_tot)-1) - columns_not_covered=list(set(all_columns) - set(columns_covered_by_mothers)) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * (len(columns_not_covered) + len(x_start_without_mother))) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) + columns_covered_by_mothers.update( + range(x_start_without_mother[dj], + x_end_without_mother[dj])) + columns_not_covered = list(all_columns - columns_covered_by_mothers) + y_type_2 = np.append(y_type_2, np.ones(len(columns_not_covered) + len(x_start_without_mother), + dtype=int) * splitter_y_new[i]) + ##y_lines_by_order = np.append(y_lines_by_order, [splitter_y_new[i]] * len(columns_not_covered)) ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) + x_starting = np.append(x_starting, np.array(columns_not_covered, int)) x_starting = np.append(x_starting, x_start_without_mother) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) + x_ending = np.append(x_ending, np.array(columns_not_covered, int) + 1) x_ending = np.append(x_ending, x_end_without_mother) - columns_covered_by_with_child_no_mothers = [] + columns_covered_by_with_child_no_mothers = set() for dj in range(len(x_end_with_child_without_mother)): - columns_covered_by_with_child_no_mothers = columns_covered_by_with_child_no_mothers + \ - list(range(x_start_with_child_without_mother[dj], - x_end_with_child_without_mother[dj])) - columns_covered_by_with_child_no_mothers = list(set(columns_covered_by_with_child_no_mothers)) - - all_columns = np.arange(len(peaks_neg_tot)-1) - columns_not_covered_child_no_mother = list(set(all_columns) - set(columns_covered_by_with_child_no_mothers)) + columns_covered_by_with_child_no_mothers.update( + range(x_start_with_child_without_mother[dj], + x_end_with_child_without_mother[dj])) + columns_not_covered_child_no_mother = list(all_columns - columns_covered_by_with_child_no_mothers) #indexes_to_be_spanned=[] for i_s in range(len(x_end_with_child_without_mother)): columns_not_covered_child_no_mother.append(x_start_with_child_without_mother[i_s]) columns_not_covered_child_no_mother = np.sort(columns_not_covered_child_no_mother) ind_args = np.arange(len(y_type_2)) - x_end_with_child_without_mother = np.array(x_end_with_child_without_mother) - x_start_with_child_without_mother = np.array(x_start_with_child_without_mother) + x_end_with_child_without_mother = np.array(x_end_with_child_without_mother, int) + x_start_with_child_without_mother = np.array(x_start_with_child_without_mother, int) for i_s_nc in columns_not_covered_child_no_mother: if i_s_nc in x_start_with_child_without_mother: x_end_biggest_column = x_end_with_child_without_mother[x_start_with_child_without_mother==i_s_nc][0] @@ -1951,7 +1935,7 @@ def return_boxes_of_images_by_order_of_reading_new( for i_c in range(len(y_column_nc)): if i_c==(len(y_column_nc)-1): ind_all_lines_between_nm_wc=ind_args[(y_type_2>y_column_nc[i_c]) & - (y_type_2=i_s_nc) & (x_ending<=x_end_biggest_column)] else: @@ -1967,21 +1951,19 @@ def return_boxes_of_images_by_order_of_reading_new( if len(x_diff_all_between_nm_wc)>0: biggest=np.argmax(x_diff_all_between_nm_wc) - columns_covered_by_mothers = [] + columns_covered_by_mothers = set() for dj in range(len(x_starting_all_between_nm_wc)): - columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_starting_all_between_nm_wc[dj], - x_ending_all_between_nm_wc[dj])) - columns_covered_by_mothers = list(set(columns_covered_by_mothers)) - - all_columns=np.arange(i_s_nc, x_end_biggest_column) - columns_not_covered = list(set(all_columns) - set(columns_covered_by_mothers)) + columns_covered_by_mothers.update( + range(x_starting_all_between_nm_wc[dj], + x_ending_all_between_nm_wc[dj])) + child_columns = set(range(i_s_nc, x_end_biggest_column)) + columns_not_covered = list(child_columns - columns_covered_by_mothers) should_longest_line_be_extended=0 if (len(x_diff_all_between_nm_wc) > 0 and set(list(range(x_starting_all_between_nm_wc[biggest], x_ending_all_between_nm_wc[biggest])) + - list(columns_not_covered)) != set(all_columns)): + list(columns_not_covered)) != child_columns): should_longest_line_be_extended=1 index_lines_so_close_to_top_separator = \ np.arange(len(y_all_between_nm_wc))[(y_all_between_nm_wc>y_column_nc[i_c]) & @@ -2008,8 +1990,8 @@ def return_boxes_of_images_by_order_of_reading_new( pass y_all_between_nm_wc = np.append(y_all_between_nm_wc, [y_column_nc[i_c]] * len(columns_not_covered)) - x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, columns_not_covered) - x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, np.array(columns_not_covered) + 1) + x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, np.array(columns_not_covered, int)) + x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, np.array(columns_not_covered, int) + 1) ind_args_between=np.arange(len(x_ending_all_between_nm_wc)) for column in range(i_s_nc, x_end_biggest_column): @@ -2078,7 +2060,7 @@ def return_boxes_of_images_by_order_of_reading_new( if len(y_in_cols)>0: y_down=np.min(y_in_cols) else: - y_down=[int(splitter_y_new[i+1])][0] + y_down=splitter_y_new[i+1] #print(y_itself,'y_itself') boxes.append([peaks_neg_tot[column], peaks_neg_tot[column+1], @@ -2086,45 +2068,42 @@ def return_boxes_of_images_by_order_of_reading_new( y_down]) except: boxes.append([0, peaks_neg_tot[len(peaks_neg_tot)-1], - int(splitter_y_new[i]), int(splitter_y_new[i+1])]) + splitter_y_new[i], splitter_y_new[i+1]]) else: y_lines_by_order=[] x_start_by_order=[] x_end_by_order=[] if len(x_starting)>0: - all_columns = np.arange(len(peaks_neg_tot)-1) - columns_covered_by_lines_covered_more_than_2col = [] + columns_covered_by_lines_covered_more_than_2col = set() for dj in range(len(x_starting)): - if set(list(range(x_starting[dj],x_ending[dj]))) == set(all_columns): - pass - else: - columns_covered_by_lines_covered_more_than_2col = columns_covered_by_lines_covered_more_than_2col + \ - list(range(x_starting[dj],x_ending[dj])) - columns_covered_by_lines_covered_more_than_2col = list(set(columns_covered_by_lines_covered_more_than_2col)) - columns_not_covered = list(set(all_columns) - set(columns_covered_by_lines_covered_more_than_2col)) + if set(range(x_starting[dj], x_ending[dj])) != all_columns: + columns_covered_by_lines_covered_more_than_2col.update( + range(x_starting[dj], x_ending[dj])) + columns_not_covered = list(all_columns - columns_covered_by_lines_covered_more_than_2col) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * (len(columns_not_covered) + 1)) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) + y_type_2 = np.append(y_type_2, np.ones(len(columns_not_covered) + 1, + dtype=int) * splitter_y_new[i]) + ##y_lines_by_order = np.append(y_lines_by_order, [splitter_y_new[i]] * len(columns_not_covered)) ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) + x_starting = np.append(x_starting, np.array(columns_not_covered, x_starting.dtype)) + x_ending = np.append(x_ending, np.array(columns_not_covered, x_ending.dtype) + 1) if len(new_main_sep_y) > 0: x_starting = np.append(x_starting, 0) - x_ending = np.append(x_ending, len(peaks_neg_tot)-1) + x_ending = np.append(x_ending, len(peaks_neg_tot) - 1) else: x_starting = np.append(x_starting, x_starting[0]) x_ending = np.append(x_ending, x_ending[0]) else: - all_columns = np.arange(len(peaks_neg_tot)-1) - columns_not_covered = list(set(all_columns)) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * len(columns_not_covered)) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) + columns_not_covered = list(all_columns) + y_type_2 = np.append(y_type_2, np.ones(len(columns_not_covered), + dtype=int) * splitter_y_new[i]) + ##y_lines_by_order = np.append(y_lines_by_order, [splitter_y_new[i]] * len(columns_not_covered)) ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) + x_starting = np.append(x_starting, np.array(columns_not_covered, x_starting.dtype)) + x_ending = np.append(x_ending, np.array(columns_not_covered, x_ending.dtype) + 1) - ind_args=np.array(range(len(y_type_2))) - #ind_args=np.array(ind_args) + ind_args = np.arange(len(y_type_2)) + for column in range(len(peaks_neg_tot)-1): #print(column,'column') ind_args_in_col=ind_args[x_starting==column] @@ -2155,7 +2134,6 @@ def return_boxes_of_images_by_order_of_reading_new( x_start_itself=x_start_copy.pop(il) x_end_itself=x_end_copy.pop(il) - #print(y_copy,'y_copy2') for column in range(x_start_itself, x_end_itself+1): #print(column,'cols') y_in_cols=[] @@ -2170,7 +2148,7 @@ def return_boxes_of_images_by_order_of_reading_new( if len(y_in_cols)>0: y_down=np.min(y_in_cols) else: - y_down=[int(splitter_y_new[i+1])][0] + y_down=splitter_y_new[i+1] #print(y_itself,'y_itself') boxes.append([peaks_neg_tot[column], peaks_neg_tot[column+1], From 66b2bce8b9f420895b8c47ebf46faf1ca3bbdd03 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 19 Sep 2025 12:19:58 +0200 Subject: [PATCH 252/374] return_boxes_of_images_by_order_of_reading_new: log any exceptions --- src/eynollah/eynollah.py | 6 ++++-- src/eynollah/utils/__init__.py | 22 ++++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 959e9a6..8080035 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4553,11 +4553,13 @@ class Eynollah: if np.abs(slope_deskew) < SLOPE_THRESHOLD: boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new( splitter_y_new, regions_without_separators, matrix_of_lines_ch, - num_col_classifier, erosion_hurts, self.tables, self.right2left) + num_col_classifier, erosion_hurts, self.tables, self.right2left, + logger=self.logger) else: boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new( splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, - num_col_classifier, erosion_hurts, self.tables, self.right2left) + num_col_classifier, erosion_hurts, self.tables, self.right2left, + logger=self.logger) if self.plotter: self.plotter.write_images_into_directory(polygons_of_images, image_page) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 7168d95..3c130d7 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1,3 +1,5 @@ +from typing import Tuple +from logging import getLogger import time import math @@ -1626,10 +1628,16 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, def return_boxes_of_images_by_order_of_reading_new( splitter_y_new, regions_without_separators, matrix_of_lines_ch, - num_col_classifier, erosion_hurts, tables, right2left_readingorder): + num_col_classifier, erosion_hurts, tables, + right2left_readingorder, + logger=None): if right2left_readingorder: regions_without_separators = cv2.flip(regions_without_separators,1) + if logger is None: + logger = getLogger(__package__) + logger.debug('enter return_boxes_of_images_by_order_of_reading_new') + boxes=[] peaks_neg_tot_tables = [] splitter_y_new = np.array(splitter_y_new, dtype=int) @@ -1710,7 +1718,7 @@ def return_boxes_of_images_by_order_of_reading_new( #print(peaks_neg_fin,'peaks_neg_fin') except: - pass + logger.exception("cannot find peaks consistent with columns") #num_col, peaks_neg_fin = find_num_col( # regions_without_separators[splitter_y_new[i]:splitter_y_new[i+1],:], # multiplier=7.0) @@ -1987,7 +1995,7 @@ def return_boxes_of_images_by_order_of_reading_new( x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, x_starting_all_between_nm_wc[biggest]) x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, x_ending_all_between_nm_wc[biggest]) except: - pass + logger.exception("cannot append") y_all_between_nm_wc = np.append(y_all_between_nm_wc, [y_column_nc[i_c]] * len(columns_not_covered)) x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, np.array(columns_not_covered, int)) @@ -2067,6 +2075,7 @@ def return_boxes_of_images_by_order_of_reading_new( y_itself, y_down]) except: + logger.exception("cannot assign boxes") boxes.append([0, peaks_neg_tot[len(peaks_neg_tot)-1], splitter_y_new[i], splitter_y_new[i+1]]) else: @@ -2170,6 +2179,7 @@ def return_boxes_of_images_by_order_of_reading_new( x_end_new = regions_without_separators.shape[1] - boxes[i][0] boxes[i][0] = x_start_new boxes[i][1] = x_end_new - return boxes, peaks_neg_tot_tables_new - else: - return boxes, peaks_neg_tot_tables + peaks_neg_tot_tables = peaks_neg_tot_tables_new + + logger.debug('exit return_boxes_of_images_by_order_of_reading_new') + return boxes, peaks_neg_tot_tables From afba70c920b4f1dc80bd70511a07df82439e6db3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 19 Aug 2025 22:56:36 +0200 Subject: [PATCH 253/374] separate_lines/do_work_of_slopes: skip if crop is empty --- src/eynollah/utils/separate_lines.py | 46 +++++++++++++++------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 0322579..ffbfff7 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1345,24 +1345,26 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest return contours_rotated_clean -def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, plotter=None): +def separate_lines_new2(img_crop, thetha, num_col, slope_region, logger=None, plotter=None): if logger is None: logger = getLogger(__package__) + if not np.prod(img_crop.shape): + return img_crop if num_col == 1: - num_patches = int(img_path.shape[1] / 200.0) + num_patches = int(img_crop.shape[1] / 200.0) else: - num_patches = int(img_path.shape[1] / 140.0) - # num_patches=int(img_path.shape[1]/200.) + num_patches = int(img_crop.shape[1] / 140.0) + # num_patches=int(img_crop.shape[1]/200.) if num_patches == 0: num_patches = 1 - img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[15]+dis_down ,:] + img_patch_interest = img_crop[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[15]+dis_down ,:] - # plt.imshow(img_patch_ineterst) + # plt.imshow(img_patch_interest) # plt.show() - length_x = int(img_path.shape[1] / float(num_patches)) + length_x = int(img_crop.shape[1] / float(num_patches)) # margin = int(0.04 * length_x) just recently this was changed because it break lines into 2 margin = int(0.04 * length_x) # if margin<=4: @@ -1370,7 +1372,7 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl # margin=0 width_mid = length_x - 2 * margin - nxf = img_path.shape[1] / float(width_mid) + nxf = img_crop.shape[1] / float(width_mid) if nxf > int(nxf): nxf = int(nxf) + 1 @@ -1386,12 +1388,12 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl index_x_d = i * width_mid index_x_u = index_x_d + length_x - if index_x_u > img_path.shape[1]: - index_x_u = img_path.shape[1] - index_x_d = img_path.shape[1] - length_x + if index_x_u > img_crop.shape[1]: + index_x_u = img_crop.shape[1] + index_x_d = img_crop.shape[1] - length_x # img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + img_xline = img_patch_interest[:, index_x_d:index_x_u] try: assert img_xline.any() @@ -1407,9 +1409,9 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl img_line_rotated = rotate_image(img_xline, slope_xline) img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 - img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] + img_patch_interest = img_crop[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] - img_patch_ineterst_revised = np.zeros(img_patch_ineterst.shape) + img_patch_interest_revised = np.zeros(img_patch_interest.shape) for i in range(nxf): if i == 0: @@ -1419,11 +1421,11 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl index_x_d = i * width_mid index_x_u = index_x_d + length_x - if index_x_u > img_path.shape[1]: - index_x_u = img_path.shape[1] - index_x_d = img_path.shape[1] - length_x + if index_x_u > img_crop.shape[1]: + index_x_u = img_crop.shape[1] + index_x_d = img_crop.shape[1] - length_x - img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + img_xline = img_patch_interest[:, index_x_d:index_x_u] img_int = np.zeros((img_xline.shape[0], img_xline.shape[1])) img_int[:, :] = img_xline[:, :] # img_patch_org[:,:,0] @@ -1446,9 +1448,9 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl int(img_int.shape[1] * (1.0)) : int(img_int.shape[1] * (1.0)) + img_int.shape[1]] img_patch_separated_returned_true_size = img_patch_separated_returned_true_size[:, margin : length_x - margin] - img_patch_ineterst_revised[:, index_x_d + margin : index_x_u - margin] = img_patch_separated_returned_true_size + img_patch_interest_revised[:, index_x_d + margin : index_x_u - margin] = img_patch_separated_returned_true_size - return img_patch_ineterst_revised + return img_patch_interest_revised def do_image_rotation(angle, img, sigma_des, logger=None): if logger is None: @@ -1546,7 +1548,7 @@ def do_work_of_slopes_new( img_int_p = all_text_region_raw[:,:] img_int_p = cv2.erode(img_int_p, KERNEL, iterations=2) - if img_int_p.shape[0] /img_int_p.shape[1] < 0.1: + if not np.prod(img_int_p.shape) or img_int_p.shape[0] /img_int_p.shape[1] < 0.1: slope = 0 slope_for_all = slope_deskew all_text_region_raw = textline_mask_tot_ea[y: y + h, x: x + w] @@ -1603,7 +1605,7 @@ def do_work_of_slopes_new_curved( # plt.imshow(img_int_p) # plt.show() - if img_int_p.shape[0] / img_int_p.shape[1] < 0.1: + if not np.prod(img_int_p.shape) or img_int_p.shape[0] / img_int_p.shape[1] < 0.1: slope = 0 slope_for_all = slope_deskew else: From 41cc38c51aaa74fb27854a101e9fbe727478f86b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 20 Aug 2025 14:28:14 +0200 Subject: [PATCH 254/374] get_textregion_contours_in_org_image_light: no back rotation, drop slope_first (always 0) --- src/eynollah/eynollah.py | 14 ++++++-------- src/eynollah/utils/contour.py | 26 +++++++++++--------------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 8080035..49f6b33 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2927,12 +2927,10 @@ class Eynollah: #print(textline_mask_tot_ea.shape, 'textline_mask_tot_ea deskew') slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True, map=self.executor.map, logger=self.logger, plotter=self.plotter) - slope_first = 0 - if self.plotter: self.plotter.save_deskewed_image(slope_deskew) self.logger.info("slope_deskew: %.2f°", slope_deskew) - return slope_deskew, slope_first + return slope_deskew def run_marginals( self, image_page, textline_mask_tot_ea, mask_images, mask_lines, @@ -4173,9 +4171,9 @@ class Eynollah: textline_mask_tot_ea_deskew = resize_image(textline_mask_tot_ea,img_h_new, img_w_new ) - slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea_deskew) + slope_deskew = self.run_deskew(textline_mask_tot_ea_deskew) else: - slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) + slope_deskew = self.run_deskew(textline_mask_tot_ea) #print("text region early -2,5 in %.1fs", time.time() - t0) #self.logger.info("Textregion detection took %.1fs ", time.time() - t1t) num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ @@ -4216,7 +4214,7 @@ class Eynollah: textline_mask_tot_ea = self.run_textline(image_page) self.logger.info("textline detection took %.1fs", time.time() - t1) t1 = time.time() - slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) + slope_deskew = self.run_deskew(textline_mask_tot_ea) self.logger.info("deskewing took %.1fs", time.time() - t1) elif num_col_classifier in (1,2): org_h_l_m = textline_mask_tot_ea.shape[0] @@ -4405,12 +4403,12 @@ class Eynollah: contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, marginal_cnts=polygons_of_marginals) #print("text region early 3.5 in %.1fs", time.time() - t0) txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( - contours_only_text_parent, self.image, slope_first, confidence_matrix, map=self.executor.map) + contours_only_text_parent, self.image, confidence_matrix) #txt_con_org = self.dilate_textregions_contours(txt_con_org) #contours_only_text_parent = self.dilate_textregions_contours(contours_only_text_parent) else: txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( - contours_only_text_parent, self.image, slope_first, confidence_matrix, map=self.executor.map) + contours_only_text_parent, self.image, confidence_matrix) #print("text region early 4 in %.1fs", time.time() - t0) boxes_text, _ = get_text_region_boxes_by_given_contours(contours_only_text_parent) boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 3d7e5c8..249748a 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -247,23 +247,19 @@ def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) return cont_int[0], index_r_con, confidence_contour -def get_textregion_contours_in_org_image_light(cnts, img, slope_first, confidence_matrix, map=map): +def get_textregion_contours_in_org_image_light(cnts, img, confidence_matrix): if not len(cnts): return [], [] - - confidence_matrix = cv2.resize(confidence_matrix, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) - img = cv2.resize(img, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) - ##cnts = list( (np.array(cnts)/2).astype(np.int16) ) - #cnts = cnts/2 - cnts = [(i/6).astype(int) for i in cnts] - results = map(partial(do_back_rotation_and_get_cnt_back, - img=img, - slope_first=slope_first, - confidence_matrix=confidence_matrix, - ), - cnts, range(len(cnts))) - contours, indexes, conf_contours = tuple(zip(*results)) - return [i*6 for i in contours], list(conf_contours) + + confidence_matrix = cv2.resize(confidence_matrix, + (img.shape[1] // 6, img.shape[0] // 6), + interpolation=cv2.INTER_NEAREST) + confs = [] + for cnt in cnts: + cnt_mask = np.zeros(confidence_matrix.shape) + cnt_mask = cv2.fillPoly(cnt_mask, pts=[cnt // 6], color=1.0) + confs.append(np.sum(confidence_matrix * cnt_mask) / np.sum(cnt_mask)) + return cnts, confs def return_contours_of_interested_textline(region_pre_p, pixel): # pixels of images are identified by 5 From 7b51fd662497ecd7c35b09764df2ed5c6b651a76 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 01:03:46 +0200 Subject: [PATCH 255/374] avoid creating invalid polygons via rounding --- src/eynollah/eynollah.py | 5 +++-- src/eynollah/utils/contour.py | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 49f6b33..0f458b4 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3670,16 +3670,17 @@ class Eynollah: return x_differential_new def dilate_textregions_contours_textline_version(self, all_found_textline_polygons): - return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords, + return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], dtype=int)[:, np.newaxis] for poly in region] for region in all_found_textline_polygons] def dilate_textregions_contours(self, all_found_textline_polygons): - return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords, + return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], dtype=int)[:, np.newaxis] for poly in all_found_textline_polygons] + def dilate_textline_contours(self, all_found_textline_polygons): for j in range(len(all_found_textline_polygons)): for ij in range(len(all_found_textline_polygons[j])): diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 249748a..8205c2b 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -49,7 +49,7 @@ def filter_contours_area_of_image(image, contours, hierarchy, max_area, min_area area <= max_area * np.prod(image.shape[:2]) and hierarchy[0][jv][3] == -1): found_polygons_early.append(np.array([[point] - for point in polygon.exterior.coords], dtype=np.uint)) + for point in polygon.exterior.coords[:-1]], dtype=np.uint)) return found_polygons_early def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, min_area): @@ -70,7 +70,7 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m True): # print(c[0][0][1]) found_polygons_early.append(np.array([[point] - for point in polygon.exterior.coords], dtype=np.int32)) + for point in polygon.exterior.coords[:-1]], dtype=np.int32)) return found_polygons_early def find_new_features_of_contours(contours_main): @@ -330,6 +330,11 @@ def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, def make_valid(polygon: Polygon) -> Polygon: """Ensures shapely.geometry.Polygon object is valid by repeated rearrangement/simplification/enlargement.""" + def isint(x): + return isinstance(x, int) or int(x) == x + # make sure rounding does not invalidate + if not all(map(isint, np.array(polygon.exterior.coords).flat)) and polygon.minimum_clearance < 1.0: + polygon = Polygon(np.round(polygon.exterior.coords)) points = list(polygon.exterior.coords) # try by re-arranging points for split in range(1, len(points)): From e730725da3d40cfbd20f857c36843190713725ca Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 01:05:15 +0200 Subject: [PATCH 256/374] check_any_text_region_in_model_one_is_main_or_header_light: return original instead of resampled contours --- src/eynollah/utils/__init__.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 3c130d7..c479744 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -957,11 +957,11 @@ def check_any_text_region_in_model_one_is_main_or_header_light( regions_model_full = cv2.resize(regions_model_full, (regions_model_full.shape[1] // zoom, regions_model_full.shape[0] // zoom), interpolation=cv2.INTER_NEAREST) - contours_only_text_parent = [(i / zoom).astype(int) for i in contours_only_text_parent] + contours_only_text_parent_z = [(cnt / zoom).astype(int) for cnt in contours_only_text_parent] ### cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin = \ - find_new_features_of_contours(contours_only_text_parent) + find_new_features_of_contours(contours_only_text_parent_z) length_con=x_max_main-x_min_main height_con=y_max_main-y_min_main @@ -984,8 +984,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light( contours_only_text_parent_main_d=[] contours_only_text_parent_head_d=[] - for ii in range(len(contours_only_text_parent)): - con=contours_only_text_parent[ii] + for ii, con in enumerate(contours_only_text_parent_z): img=np.zeros((regions_model_1.shape[0], regions_model_1.shape[1], 3)) img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) @@ -996,23 +995,22 @@ def check_any_text_region_in_model_one_is_main_or_header_light( if (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 - contours_only_text_parent_head.append(con) + contours_only_text_parent_head.append(contours_only_text_parent[ii]) + conf_contours_head.append(None) # why not conf_contours[ii], too? if contours_only_text_parent_d_ordered is not None: contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_head.append(all_box_coord[ii]) slopes_head.append(slopes[ii]) all_found_textline_polygons_head.append(all_found_textline_polygons[ii]) - conf_contours_head.append(None) else: regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 - contours_only_text_parent_main.append(con) + contours_only_text_parent_main.append(contours_only_text_parent[ii]) conf_contours_main.append(conf_contours[ii]) if contours_only_text_parent_d_ordered is not None: contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_main.append(all_box_coord[ii]) slopes_main.append(slopes[ii]) all_found_textline_polygons_main.append(all_found_textline_polygons[ii]) - #print(all_pixels,pixels_main,pixels_header) ### to make it faster @@ -1020,8 +1018,6 @@ def check_any_text_region_in_model_one_is_main_or_header_light( # regions_model_full = cv2.resize(img, (regions_model_full.shape[1] // zoom, # regions_model_full.shape[0] // zoom), # interpolation=cv2.INTER_NEAREST) - contours_only_text_parent_head = [(i * zoom).astype(int) for i in contours_only_text_parent_head] - contours_only_text_parent_main = [(i * zoom).astype(int) for i in contours_only_text_parent_main] ### return (regions_model_1, From 17bcf1af71802d790f7508d52221d64ea4fff939 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 01:32:32 +0200 Subject: [PATCH 257/374] =?UTF-8?q?rename=20*lines=5Fxml=20=E2=86=92=20*se?= =?UTF-8?q?plines=20for=20clarity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/eynollah/eynollah.py | 58 ++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0f458b4..c04c481 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1713,9 +1713,9 @@ class Eynollah: mask_texts_only = (prediction_regions_org[:,:] ==1)*1 mask_images_only=(prediction_regions_org[:,:] ==2)*1 - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - polygons_lines_xml = textline_con_fil = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) + polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) + polygons_seplines = textline_con_fil = filter_contours_area_of_image( + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) @@ -1779,7 +1779,7 @@ class Eynollah: [page_coord_img[2], page_coord_img[1]]])) self.logger.debug("exit get_regions_extract_images_only") - return text_regions_p_true, erosion_hurts, polygons_lines_xml, polygons_of_images_fin, image_page, page_coord, cont_page + return text_regions_p_true, erosion_hurts, polygons_seplines, polygons_of_images_fin, image_page, page_coord, cont_page def get_regions_light_v(self,img,is_image_enhanced, num_col_classifier, skip_layout_and_reading_order=False): self.logger.debug("enter get_regions_light_v") @@ -1895,24 +1895,24 @@ class Eynollah: mask_texts_only = cv2.dilate(mask_texts_only, kernel=np.ones((2,2), np.uint8), iterations=1) mask_images_only=(prediction_regions_org[:,:] ==2)*1 - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) + polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) test_khat = np.zeros(prediction_regions_org.shape) - test_khat = cv2.fillPoly(test_khat, pts=polygons_lines_xml, color=(1,1,1)) + test_khat = cv2.fillPoly(test_khat, pts=polygons_seplines, color=(1,1,1)) #plt.imshow(test_khat[:,:]) #plt.show() #for jv in range(1): - #print(jv, hir_lines_xml[0][232][3]) + #print(jv, hir_seplines[0][232][3]) #test_khat = np.zeros(prediction_regions_org.shape) - #test_khat = cv2.fillPoly(test_khat, pts = [polygons_lines_xml[232]], color=(1,1,1)) + #test_khat = cv2.fillPoly(test_khat, pts = [polygons_seplines[232]], color=(1,1,1)) #plt.imshow(test_khat[:,:]) #plt.show() - polygons_lines_xml = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) + polygons_seplines = filter_contours_area_of_image( + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) test_khat = np.zeros(prediction_regions_org.shape) - test_khat = cv2.fillPoly(test_khat, pts = polygons_lines_xml, color=(1,1,1)) + test_khat = cv2.fillPoly(test_khat, pts = polygons_seplines, color=(1,1,1)) #plt.imshow(test_khat[:,:]) #plt.show() @@ -1937,7 +1937,7 @@ class Eynollah: #plt.show() #print("inside 4 ", time.time()-t_in) self.logger.debug("exit get_regions_light_v") - return text_regions_p_true, erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin, confidence_matrix + return text_regions_p_true, erosion_hurts, polygons_seplines, textline_mask_tot_ea, img_bin, confidence_matrix else: img_bin = resize_image(img_bin,img_height_h, img_width_h ) self.logger.debug("exit get_regions_light_v") @@ -2020,9 +2020,9 @@ class Eynollah: mask_texts_only=(prediction_regions_org[:,:]==1)*1 mask_images_only=(prediction_regions_org[:,:]==2)*1 - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - polygons_lines_xml = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) + polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) + polygons_seplines = filter_contours_area_of_image( + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only, 1, 0.00001) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only, 1, 0.00001) @@ -2034,7 +2034,7 @@ class Eynollah: text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) self.logger.debug("exit get_regions_from_xy_2models") - return text_regions_p_true, erosion_hurts, polygons_lines_xml + return text_regions_p_true, erosion_hurts, polygons_seplines except: if self.input_binary: prediction_bin = np.copy(img_org) @@ -2069,9 +2069,9 @@ class Eynollah: mask_texts_only = (prediction_regions_org == 1)*1 mask_images_only= (prediction_regions_org == 2)*1 - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - polygons_lines_xml = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) + polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) + polygons_seplines = filter_contours_area_of_image( + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) @@ -2084,7 +2084,7 @@ class Eynollah: erosion_hurts = True self.logger.debug("exit get_regions_from_xy_2models") - return text_regions_p_true, erosion_hurts, polygons_lines_xml + return text_regions_p_true, erosion_hurts, polygons_seplines def do_order_of_regions_full_layout( self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): @@ -4102,7 +4102,7 @@ class Eynollah: img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) self.logger.info("Enhancing took %.1fs ", time.time() - t0) if self.extract_only_images: - text_regions_p_1, erosion_hurts, polygons_lines_xml, polygons_of_images, image_page, page_coord, cont_page = \ + text_regions_p_1, erosion_hurts, polygons_seplines, polygons_of_images, image_page, page_coord, cont_page = \ self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( @@ -4145,7 +4145,7 @@ class Eynollah: polygons_of_marginals = [] all_found_textline_polygons_marginals = [] all_box_coord_marginals = [] - polygons_lines_xml = [] + polygons_seplines = [] contours_tables = [] ocr_all_textlines = None conf_contours_textregions =None @@ -4153,13 +4153,13 @@ class Eynollah: cont_page, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + cont_page, polygons_seplines, contours_tables, ocr_all_textlines, conf_contours_textregions) return pcgts #print("text region early -1 in %.1fs", time.time() - t0) t1 = time.time() if self.light_version: - text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ + text_regions_p_1, erosion_hurts, polygons_seplines, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier) #print("text region early -2 in %.1fs", time.time() - t0) @@ -4186,7 +4186,7 @@ class Eynollah: textline_mask_tot_ea_org = np.copy(textline_mask_tot_ea) #print("text region early -4 in %.1fs", time.time() - t0) else: - text_regions_p_1 ,erosion_hurts, polygons_lines_xml = \ + text_regions_p_1, erosion_hurts, polygons_seplines = \ self.get_regions_from_xy_2models(img_res, is_image_enhanced, num_col_classifier) self.logger.info("Textregion detection took %.1fs ", time.time() - t1) @@ -4385,13 +4385,13 @@ class Eynollah: [], [], page_coord, [], [], [], [], [], [], polygons_of_images, contours_tables, [], polygons_of_marginals, empty_marginals, empty_marginals, [], [], [], - cont_page, polygons_lines_xml, [], [], []) + cont_page, polygons_seplines, [], [], []) else: pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], polygons_of_images, polygons_of_marginals, empty_marginals, empty_marginals, [], [], - cont_page, polygons_lines_xml, contours_tables, [], []) + cont_page, polygons_seplines, contours_tables, [], []) return pcgts @@ -4586,7 +4586,7 @@ class Eynollah: all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, - cont_page, polygons_lines_xml, ocr_all_textlines, conf_contours_textregions, conf_contours_textregions_h) + cont_page, polygons_seplines, ocr_all_textlines, conf_contours_textregions, conf_contours_textregions_h) return pcgts contours_only_text_parent_h = None @@ -4665,7 +4665,7 @@ class Eynollah: txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) + cont_page, polygons_seplines, contours_tables, ocr_all_textlines, conf_contours_textregions) return pcgts From a433c736281dcf86630f80bfa686064814b313d9 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 01:33:16 +0200 Subject: [PATCH 258/374] filter_contours_area_of_image*: also ensure validity here --- src/eynollah/eynollah.py | 4 ++-- src/eynollah/utils/contour.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index c04c481..7b3b81a 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3671,13 +3671,13 @@ class Eynollah: def dilate_textregions_contours_textline_version(self, all_found_textline_polygons): return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], - dtype=int)[:, np.newaxis] + dtype=np.uint)[:, np.newaxis] for poly in region] for region in all_found_textline_polygons] def dilate_textregions_contours(self, all_found_textline_polygons): return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], - dtype=int)[:, np.newaxis] + dtype=np.uint)[:, np.newaxis] for poly in all_found_textline_polygons] diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 8205c2b..03d45b7 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -48,8 +48,8 @@ def filter_contours_area_of_image(image, contours, hierarchy, max_area, min_area if (area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and hierarchy[0][jv][3] == -1): - found_polygons_early.append(np.array([[point] - for point in polygon.exterior.coords[:-1]], dtype=np.uint)) + found_polygons_early.append(np.array(make_valid(polygon).exterior.coords[:-1], + dtype=np.uint)[:, np.newaxis]) return found_polygons_early def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, min_area): @@ -69,8 +69,8 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m # hierarchy[0][jv][3]==-1 True): # print(c[0][0][1]) - found_polygons_early.append(np.array([[point] - for point in polygon.exterior.coords[:-1]], dtype=np.int32)) + found_polygons_early.append(np.array(make_valid(polygon).exterior.coords[:-1], + dtype=np.uint)[:, np.newaxis]) return found_polygons_early def find_new_features_of_contours(contours_main): From 0650274ffad576acde6048822b5f74b6303ef689 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 01:42:46 +0200 Subject: [PATCH 259/374] =?UTF-8?q?move=20dilate=5F*=5Fcontours=20to=20.ut?= =?UTF-8?q?ils.contour,=20rename=20dilate=5Ftextregions=5Fcontours=5Ftextl?= =?UTF-8?q?ine=5Fversion=20=E2=86=92=20dilate=5Ftextline=5Fcontours?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/eynollah/eynollah.py | 253 ++-------------------------------- src/eynollah/utils/contour.py | 11 ++ 2 files changed, 22 insertions(+), 242 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 7b3b81a..fe233cb 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -69,12 +69,13 @@ from .utils.contour import ( get_text_region_boxes_by_given_contours, get_textregion_contours_in_org_image, get_textregion_contours_in_org_image_light, - make_valid, return_contours_of_image, return_contours_of_interested_region, return_contours_of_interested_region_by_min_size, return_contours_of_interested_textline, return_parent_contours, + dilate_textregion_contours, + dilate_textline_contours, ) from .utils.rotate import ( rotate_image, @@ -1919,7 +1920,7 @@ class Eynollah: #sys.exit() polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) - ##polygons_of_only_texts = self.dilate_textregions_contours(polygons_of_only_texts) + ##polygons_of_only_texts = dilate_textregion_contours(polygons_of_only_texts) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) text_regions_p_true = np.zeros(prediction_regions_org.shape) @@ -3669,117 +3670,6 @@ class Eynollah: return x_differential_new - def dilate_textregions_contours_textline_version(self, all_found_textline_polygons): - return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], - dtype=np.uint)[:, np.newaxis] - for poly in region] - for region in all_found_textline_polygons] - - def dilate_textregions_contours(self, all_found_textline_polygons): - return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], - dtype=np.uint)[:, np.newaxis] - for poly in all_found_textline_polygons] - - - def dilate_textline_contours(self, all_found_textline_polygons): - for j in range(len(all_found_textline_polygons)): - for ij in range(len(all_found_textline_polygons[j])): - con_ind = all_found_textline_polygons[j][ij] - area = cv2.contourArea(con_ind) - - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_differential = gaussian_filter1d(x_differential, 3) - y_differential = gaussian_filter1d(y_differential, 3) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - x_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in x_differential] - y_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in y_differential] - - abs_diff=abs(abs(x_differential)- abs(y_differential) ) - - inc_x = np.zeros(len(x_differential)+1) - inc_y = np.zeros(len(x_differential)+1) - - if (y_max-y_min) <= (x_max-x_min): - dilation_m1 = round(area / (x_max-x_min) * 0.35) - else: - dilation_m1 = round(area / (y_max-y_min) * 0.35) - - if dilation_m1>12: - dilation_m1 = 12 - if dilation_m1<4: - dilation_m1 = 4 - #print(dilation_m1, 'dilation_m1') - dilation_m2 = int(dilation_m1/2.) +1 - - for i in range(len(x_differential)): - if abs_diff[i]==0: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]==0 and y_differential_mask_nonzeros[i]!=0: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]!=0 and y_differential_mask_nonzeros[i]==0: - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - - elif abs_diff[i]!=0 and abs_diff[i]>=3: - if abs(x_differential[i])>abs(y_differential[i]): - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - else: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - else: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - - inc_x[0] = inc_x[-1] - inc_y[0] = inc_y[-1] - - con_scaled = con_ind*1 - - con_scaled[:,0, 0] = con_ind[:,0,0] + np.array(inc_x)[:] - con_scaled[:,0, 1] = con_ind[:,0,1] + np.array(inc_y)[:] - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - con_ind = con_ind.astype(np.int32) - - results = [cv2.pointPolygonTest(con_ind, (con_scaled[ind,0, 0], con_scaled[ind,0, 1]), False) - for ind in range(len(con_scaled[:,0, 1])) ] - results = np.array(results) - results[results==0] = 1 - - diff_result = np.diff(results) - - indices_2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==2] - indices_m2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==-2] - - if results[0]==1: - con_scaled[:indices_m2[0]+1,0, 1] = con_ind[:indices_m2[0]+1,0,1] - con_scaled[:indices_m2[0]+1,0, 0] = con_ind[:indices_m2[0]+1,0,0] - indices_m2 = indices_m2[1:] - - if len(indices_2)>len(indices_m2): - con_scaled[indices_2[-1]+1:,0, 1] = con_ind[indices_2[-1]+1:,0,1] - con_scaled[indices_2[-1]+1:,0, 0] = con_ind[indices_2[-1]+1:,0,0] - indices_2 = indices_2[:-1] - - for ii in range(len(indices_2)): - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 1] = con_scaled[indices_2[ii],0, 1] - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 0] = con_scaled[indices_2[ii],0, 0] - - all_found_textline_polygons[j][ij][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][ij][:,0,0] = con_scaled[:,0, 0] - return all_found_textline_polygons - def filter_contours_inside_a_bigger_one(self,contours, contours_d_ordered, image, marginal_cnts=None, type_contour="textregion"): if type_contour=="textregion": areas = [cv2.contourArea(contours[j]) for j in range(len(contours))] @@ -3917,121 +3807,6 @@ class Eynollah: return contours, text_con_org, conf_contours_textregions, contours_textline, contours_only_text_parent_d_ordered, np.array(range(len(contours))) - def dilate_textlines(self, all_found_textline_polygons): - for j in range(len(all_found_textline_polygons)): - for i in range(len(all_found_textline_polygons[j])): - con_ind = all_found_textline_polygons[j][i] - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - if (y_max - y_min) > (x_max - x_min) and (x_max - x_min)<70: - x_biger_than_x = np.abs(x_differential) > np.abs(y_differential) - mult = x_biger_than_x*x_differential - - arg_min_mult = np.argmin(mult) - arg_max_mult = np.argmax(mult) - - if y_differential[0]==0: - y_differential[0] = 0.1 - if y_differential[-1]==0: - y_differential[-1]= 0.1 - y_differential = [y_differential[ind] if y_differential[ind] != 0 - else 0.5 * (y_differential[ind-1] + y_differential[ind+1]) - for ind in range(len(y_differential))] - - if y_differential[0]==0.1: - y_differential[0] = y_differential[1] - if y_differential[-1]==0.1: - y_differential[-1] = y_differential[-2] - y_differential.append(y_differential[0]) - - y_differential = [-1 if y_differential[ind] < 0 else 1 - for ind in range(len(y_differential))] - y_differential = self.return_it_in_two_groups(y_differential) - y_differential = np.array(y_differential) - - con_scaled = con_ind*1 - con_scaled[:,0, 0] = con_ind[:,0,0] - 8*y_differential - con_scaled[arg_min_mult,0, 1] = con_ind[arg_min_mult,0,1] + 8 - con_scaled[arg_min_mult+1,0, 1] = con_ind[arg_min_mult+1,0,1] + 8 - - try: - con_scaled[arg_min_mult-1,0, 1] = con_ind[arg_min_mult-1,0,1] + 5 - con_scaled[arg_min_mult+2,0, 1] = con_ind[arg_min_mult+2,0,1] + 5 - except: - pass - - con_scaled[arg_max_mult,0, 1] = con_ind[arg_max_mult,0,1] - 8 - con_scaled[arg_max_mult+1,0, 1] = con_ind[arg_max_mult+1,0,1] - 8 - - try: - con_scaled[arg_max_mult-1,0, 1] = con_ind[arg_max_mult-1,0,1] - 5 - con_scaled[arg_max_mult+2,0, 1] = con_ind[arg_max_mult+2,0,1] - 5 - except: - pass - - else: - y_biger_than_x = np.abs(y_differential) > np.abs(x_differential) - mult = y_biger_than_x*y_differential - - arg_min_mult = np.argmin(mult) - arg_max_mult = np.argmax(mult) - - if x_differential[0]==0: - x_differential[0] = 0.1 - if x_differential[-1]==0: - x_differential[-1]= 0.1 - x_differential = [x_differential[ind] if x_differential[ind] != 0 - else 0.5 * (x_differential[ind-1] + x_differential[ind+1]) - for ind in range(len(x_differential))] - - if x_differential[0]==0.1: - x_differential[0] = x_differential[1] - if x_differential[-1]==0.1: - x_differential[-1] = x_differential[-2] - x_differential.append(x_differential[0]) - - x_differential = [-1 if x_differential[ind] < 0 else 1 - for ind in range(len(x_differential))] - x_differential = self.return_it_in_two_groups(x_differential) - x_differential = np.array(x_differential) - - con_scaled = con_ind*1 - con_scaled[:,0, 1] = con_ind[:,0,1] + 8*x_differential - con_scaled[arg_min_mult,0, 0] = con_ind[arg_min_mult,0,0] + 8 - con_scaled[arg_min_mult+1,0, 0] = con_ind[arg_min_mult+1,0,0] + 8 - - try: - con_scaled[arg_min_mult-1,0, 0] = con_ind[arg_min_mult-1,0,0] + 5 - con_scaled[arg_min_mult+2,0, 0] = con_ind[arg_min_mult+2,0,0] + 5 - except: - pass - - con_scaled[arg_max_mult,0, 0] = con_ind[arg_max_mult,0,0] - 8 - con_scaled[arg_max_mult+1,0, 0] = con_ind[arg_max_mult+1,0,0] - 8 - - try: - con_scaled[arg_max_mult-1,0, 0] = con_ind[arg_max_mult-1,0,0] - 5 - con_scaled[arg_max_mult+2,0, 0] = con_ind[arg_max_mult+2,0,0] - 5 - except: - pass - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - all_found_textline_polygons[j][i][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][i][:,0,0] = con_scaled[:,0, 0] - - return all_found_textline_polygons - def delete_regions_without_textlines( self, slopes, all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con): @@ -4130,8 +3905,7 @@ class Eynollah: all_found_textline_polygons=[ all_found_textline_polygons ] - all_found_textline_polygons = self.dilate_textregions_contours_textline_version( - all_found_textline_polygons) + all_found_textline_polygons = dilate_textline_contours(all_found_textline_polygons) all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( all_found_textline_polygons, None, textline_mask_tot_ea, type_contour="textline") @@ -4255,14 +4029,14 @@ class Eynollah: boxes, boxes_d, polygons_of_marginals, contours_tables = \ self.run_boxes_no_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, table_prediction, erosion_hurts) - ###polygons_of_marginals = self.dilate_textregions_contours(polygons_of_marginals) + ###polygons_of_marginals = dilate_textregion_contours(polygons_of_marginals) else: polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, \ regions_fully, regions_without_separators, polygons_of_marginals, contours_tables = \ self.run_boxes_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions, table_prediction, erosion_hurts, img_bin_light if self.light_version else None) - ###polygons_of_marginals = self.dilate_textregions_contours(polygons_of_marginals) + ###polygons_of_marginals = dilate_textregion_contours(polygons_of_marginals) if self.light_version: drop_label_in_full_layout = 4 textline_mask_tot_ea_org[img_revised_tab==drop_label_in_full_layout] = 0 @@ -4398,15 +4172,14 @@ class Eynollah: #print("text region early 3 in %.1fs", time.time() - t0) if self.light_version: - contours_only_text_parent = self.dilate_textregions_contours( - contours_only_text_parent) + contours_only_text_parent = dilate_textregion_contours(contours_only_text_parent) contours_only_text_parent , contours_only_text_parent_d_ordered = self.filter_contours_inside_a_bigger_one( contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, marginal_cnts=polygons_of_marginals) #print("text region early 3.5 in %.1fs", time.time() - t0) txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) - #txt_con_org = self.dilate_textregions_contours(txt_con_org) - #contours_only_text_parent = self.dilate_textregions_contours(contours_only_text_parent) + #txt_con_org = dilate_textregion_contours(txt_con_org) + #contours_only_text_parent = dilate_textregion_contours(contours_only_text_parent) else: txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) @@ -4433,14 +4206,10 @@ class Eynollah: #slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, polygons_of_marginals, polygons_of_marginals, _ = \ # self.delete_regions_without_textlines(slopes_marginals, all_found_textline_polygons_marginals, # boxes_marginals, polygons_of_marginals, polygons_of_marginals, np.array(range(len(polygons_of_marginals)))) - #all_found_textline_polygons = self.dilate_textlines(all_found_textline_polygons) - #####all_found_textline_polygons = self.dilate_textline_contours(all_found_textline_polygons) - all_found_textline_polygons = self.dilate_textregions_contours_textline_version( - all_found_textline_polygons) + all_found_textline_polygons = dilate_textline_contours(all_found_textline_polygons) all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( all_found_textline_polygons, None, textline_mask_tot_ea_org, type_contour="textline") - all_found_textline_polygons_marginals = self.dilate_textregions_contours_textline_version( - all_found_textline_polygons_marginals) + all_found_textline_polygons_marginals = dilate_textline_contours(all_found_textline_polygons_marginals) contours_only_text_parent, txt_con_org, conf_contours_textregions, all_found_textline_polygons, contours_only_text_parent_d_ordered, \ index_by_text_par_con = self.filter_contours_without_textline_inside( contours_only_text_parent, txt_con_org, all_found_textline_polygons, contours_only_text_parent_d_ordered, conf_contours_textregions) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 03d45b7..f228e53 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -328,6 +328,17 @@ def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, return img_ret[:, :, 0] +def dilate_textline_contours(self, all_found_textline_polygons): + return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], + dtype=np.uint)[:, np.newaxis] + for poly in region] + for region in all_found_textline_polygons] + +def dilate_textregion_contours(self, all_found_textline_polygons): + return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], + dtype=np.uint)[:, np.newaxis] + for poly in all_found_textline_polygons] + def make_valid(polygon: Polygon) -> Polygon: """Ensures shapely.geometry.Polygon object is valid by repeated rearrangement/simplification/enlargement.""" def isint(x): From f3faa29528ce7acdafa0c02fc2a9ec4732d91e4a Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 12:59:03 +0200 Subject: [PATCH 260/374] refactor shapely converisons into contour2polygon / polygon2contour, also handle heterogeneous geometries --- src/eynollah/eynollah.py | 1 - src/eynollah/utils/contour.py | 107 ++++++++++++++++++++++++++-------- 2 files changed, 83 insertions(+), 25 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index fe233cb..54ace30 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -27,7 +27,6 @@ from loky import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 import numpy as np -from shapely.geometry import Polygon from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d from numba import cuda diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index f228e53..1123241 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -1,7 +1,15 @@ +from typing import Sequence, Union +from numbers import Number from functools import partial +import itertools + import cv2 import numpy as np -from shapely.geometry import Polygon +from scipy.sparse.csgraph import minimum_spanning_tree +from shapely.geometry import Polygon, LineString +from shapely.geometry.polygon import orient +from shapely import set_precision +from shapely.ops import unary_union, nearest_points from .rotate import rotate_image, rotation_image_new @@ -37,29 +45,28 @@ def get_text_region_boxes_by_given_contours(contours): return boxes, contours_new -def filter_contours_area_of_image(image, contours, hierarchy, max_area, min_area): +def filter_contours_area_of_image(image, contours, hierarchy, max_area=1.0, min_area=0.0, dilate=0): found_polygons_early = [] - for jv,c in enumerate(contours): - if len(c) < 3: # A polygon cannot have less than 3 points + for jv, contour in enumerate(contours): + if len(contour) < 3: # A polygon cannot have less than 3 points continue - polygon = Polygon([point[0] for point in c]) + polygon = contour2polygon(contour, dilate=dilate) area = polygon.area if (area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and hierarchy[0][jv][3] == -1): - found_polygons_early.append(np.array(make_valid(polygon).exterior.coords[:-1], - dtype=np.uint)[:, np.newaxis]) + found_polygons_early.append(polygon2contour(polygon)) return found_polygons_early -def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, min_area): +def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area=1.0, min_area=0.0, dilate=0): found_polygons_early = [] - for jv,c in enumerate(contours): - if len(c) < 3: # A polygon cannot have less than 3 points + for jv, contour in enumerate(contours): + if len(contour) < 3: # A polygon cannot have less than 3 points continue - polygon = Polygon([point[0] for point in c]) - # area = cv2.contourArea(c) + polygon = contour2polygon(contour, dilate=dilate) + # area = cv2.contourArea(contour) area = polygon.area ##print(np.prod(thresh.shape[:2])) # Check that polygon has area greater than minimal area @@ -68,9 +75,8 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, m area <= max_area * np.prod(image.shape[:2]) and # hierarchy[0][jv][3]==-1 True): - # print(c[0][0][1]) - found_polygons_early.append(np.array(make_valid(polygon).exterior.coords[:-1], - dtype=np.uint)[:, np.newaxis]) + # print(contour[0][0][1]) + found_polygons_early.append(polygon2contour(polygon)) return found_polygons_early def find_new_features_of_contours(contours_main): @@ -328,16 +334,29 @@ def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, return img_ret[:, :, 0] -def dilate_textline_contours(self, all_found_textline_polygons): - return [[np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], - dtype=np.uint)[:, np.newaxis] - for poly in region] +def dilate_textline_contours(all_found_textline_polygons): + return [[polygon2contour(contour2polygon(contour, dilate=5)) + for contour in region] for region in all_found_textline_polygons] -def dilate_textregion_contours(self, all_found_textline_polygons): - return [np.array(make_valid(Polygon(poly[:, 0]).buffer(5)).exterior.coords[:-1], - dtype=np.uint)[:, np.newaxis] - for poly in all_found_textline_polygons] +def dilate_textregion_contours(all_found_textline_polygons): + return [polygon2contour(contour2polygon(contour, dilate=5)) + for contour in all_found_textline_polygons] + +def contour2polygon(contour: Union[np.ndarray, Sequence[Sequence[Sequence[Number]]]], dilate=0): + polygon = Polygon([point[0] for point in contour]) + if dilate: + polygon = polygon.buffer(dilate) + if polygon.geom_type == 'GeometryCollection': + # heterogeneous result: filter zero-area shapes (LineString, Point) + polygon = unary_union([geom for geom in polygon.geoms if geom.area > 0]) + if polygon.geom_type == 'MultiPolygon': + # homogeneous result: construct convex hull to connect + polygon = join_polygons(polygon.geoms) + return make_valid(polygon) + +def polygon2contour(polygon: Polygon) -> np.ndarray: + return np.array(polygon.exterior.coords[:-1], dtype=np.uint)[:, np.newaxis] def make_valid(polygon: Polygon) -> Polygon: """Ensures shapely.geometry.Polygon object is valid by repeated rearrangement/simplification/enlargement.""" @@ -346,7 +365,7 @@ def make_valid(polygon: Polygon) -> Polygon: # make sure rounding does not invalidate if not all(map(isint, np.array(polygon.exterior.coords).flat)) and polygon.minimum_clearance < 1.0: polygon = Polygon(np.round(polygon.exterior.coords)) - points = list(polygon.exterior.coords) + points = list(polygon.exterior.coords[:-1]) # try by re-arranging points for split in range(1, len(points)): if polygon.is_valid or polygon.simplify(polygon.area).is_valid: @@ -368,3 +387,43 @@ def make_valid(polygon: Polygon) -> Polygon: polygon = polygon.buffer(tolerance) assert polygon.is_valid, polygon.wkt return polygon + +def join_polygons(polygons: Sequence[Polygon], scale=20) -> Polygon: + """construct concave hull (alpha shape) from input polygons by connecting their pairwise nearest points""" + # ensure input polygons are simply typed and all oriented equally + polygons = [orient(poly) + for poly in itertools.chain.from_iterable( + [poly.geoms + if poly.geom_type in ['MultiPolygon', 'GeometryCollection'] + else [poly] + for poly in polygons])] + npoly = len(polygons) + if npoly == 1: + return polygons[0] + # find min-dist path through all polygons (travelling salesman) + pairs = itertools.combinations(range(npoly), 2) + dists = np.zeros((npoly, npoly), dtype=float) + for i, j in pairs: + dist = polygons[i].distance(polygons[j]) + if dist < 1e-5: + dist = 1e-5 # if pair merely touches, we still need to get an edge + dists[i, j] = dist + dists[j, i] = dist + dists = minimum_spanning_tree(dists, overwrite=True) + # add bridge polygons (where necessary) + for prevp, nextp in zip(*dists.nonzero()): + prevp = polygons[prevp] + nextp = polygons[nextp] + nearest = nearest_points(prevp, nextp) + bridgep = orient(LineString(nearest).buffer(max(1, scale/5), resolution=1), -1) + polygons.append(bridgep) + jointp = unary_union(polygons) + assert jointp.geom_type == 'Polygon', jointp.wkt + # follow-up calculations will necessarily be integer; + # so anticipate rounding here and then ensure validity + jointp2 = set_precision(jointp, 1.0) + if jointp2.geom_type != 'Polygon' or not jointp2.is_valid: + jointp2 = Polygon(np.round(jointp.exterior.coords)) + jointp2 = make_valid(jointp2) + assert jointp2.geom_type == 'Polygon', jointp2.wkt + return jointp2 From 7a9e8256ee8a4c777baa0bd972697cece3e269a5 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 21 Aug 2025 13:00:31 +0200 Subject: [PATCH 261/374] =?UTF-8?q?increase=20dilatation:=20textregions/li?= =?UTF-8?q?nes=20(5=E2=86=926),=20seplines=20(0=E2=86=921)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/eynollah/eynollah.py | 10 +++++----- src/eynollah/utils/contour.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 54ace30..8cb1d52 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1714,8 +1714,8 @@ class Eynollah: mask_images_only=(prediction_regions_org[:,:] ==2)*1 polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) - polygons_seplines = textline_con_fil = filter_contours_area_of_image( - mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) + polygons_seplines = filter_contours_area_of_image( + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001, dilate=1) polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) @@ -1909,7 +1909,7 @@ class Eynollah: #plt.show() polygons_seplines = filter_contours_area_of_image( - mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001, dilate=1) test_khat = np.zeros(prediction_regions_org.shape) test_khat = cv2.fillPoly(test_khat, pts = polygons_seplines, color=(1,1,1)) @@ -2022,7 +2022,7 @@ class Eynollah: polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) polygons_seplines = filter_contours_area_of_image( - mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001, dilate=1) polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only, 1, 0.00001) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only, 1, 0.00001) @@ -2071,7 +2071,7 @@ class Eynollah: polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) polygons_seplines = filter_contours_area_of_image( - mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001) + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001, dilate=1) polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 1123241..c571be6 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -335,12 +335,12 @@ def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, return img_ret[:, :, 0] def dilate_textline_contours(all_found_textline_polygons): - return [[polygon2contour(contour2polygon(contour, dilate=5)) + return [[polygon2contour(contour2polygon(contour, dilate=6)) for contour in region] for region in all_found_textline_polygons] def dilate_textregion_contours(all_found_textline_polygons): - return [polygon2contour(contour2polygon(contour, dilate=5)) + return [polygon2contour(contour2polygon(contour, dilate=6)) for contour in all_found_textline_polygons] def contour2polygon(contour: Union[np.ndarray, Sequence[Sequence[Sequence[Number]]]], dilate=0): From 11e143afee1f446bfef7c6b19ba720e5cddb981d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 29 Aug 2025 12:16:56 +0200 Subject: [PATCH 262/374] polygon2contour: avoid overflow --- src/eynollah/utils/contour.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index c571be6..2cd7080 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -356,7 +356,8 @@ def contour2polygon(contour: Union[np.ndarray, Sequence[Sequence[Sequence[Number return make_valid(polygon) def polygon2contour(polygon: Polygon) -> np.ndarray: - return np.array(polygon.exterior.coords[:-1], dtype=np.uint)[:, np.newaxis] + polygon = np.array(polygon.exterior.coords[:-1], dtype=int) + return np.maximum(0, polygon).astype(np.uint)[:, np.newaxis] def make_valid(polygon: Polygon) -> Polygon: """Ensures shapely.geometry.Polygon object is valid by repeated rearrangement/simplification/enlargement.""" From 235539a35071559f8929bfcda9cb47d506c23d58 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 29 Aug 2025 12:19:37 +0200 Subject: [PATCH 263/374] filter_contours_without_textline_inside: avoid removing from identical lists twice --- src/eynollah/eynollah.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 8cb1d52..b636b09 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3764,7 +3764,9 @@ class Eynollah: return contours def filter_contours_without_textline_inside( - self, contours,text_con_org, contours_textline, contours_only_text_parent_d_ordered, conf_contours_textregions): + self, contours, text_con_org, contours_textline, + contours_only_text_parent_d_ordered, + conf_contours_textregions): ###contours_txtline_of_all_textregions = [] ###for jj in range(len(contours_textline)): ###contours_txtline_of_all_textregions = contours_txtline_of_all_textregions + contours_textline[jj] @@ -3788,23 +3790,23 @@ class Eynollah: ###if np.any(results==1): ###contours_with_textline.append(con_tr) - textregion_index_to_del = [] + textregion_index_to_del = set() for index_textregion, textlines_textregion in enumerate(contours_textline): - if len(textlines_textregion)==0: - textregion_index_to_del.append(index_textregion) + if len(textlines_textregion) == 0: + textregion_index_to_del.add(index_textregion) + def filterfun(lis): + if len(lis) == 0: + return [] + if len(textregion_index_to_del) == 0: + return lis + return list(np.delete(lis, list(textregion_index_to_del))) - uniqe_args_trs = np.unique(textregion_index_to_del) - uniqe_args_trs_sorted = np.sort(uniqe_args_trs)[::-1] - - for ind_u_a_trs in uniqe_args_trs_sorted: - conf_contours_textregions.pop(ind_u_a_trs) - contours.pop(ind_u_a_trs) - contours_textline.pop(ind_u_a_trs) - text_con_org.pop(ind_u_a_trs) - if len(contours_only_text_parent_d_ordered) > 0: - contours_only_text_parent_d_ordered.pop(ind_u_a_trs) - - return contours, text_con_org, conf_contours_textregions, contours_textline, contours_only_text_parent_d_ordered, np.array(range(len(contours))) + return (filterfun(contours), + filterfun(text_con_org), + filterfun(conf_contours_textregions), + filterfun(contours_textline), + filterfun(contours_only_text_parent_d_ordered), + np.arange(len(contours) - len(textregion_index_to_del))) def delete_regions_without_textlines( self, slopes, all_found_textline_polygons, boxes_text, txt_con_org, From bca2ae3d78fcc6536c5365c9b93a0143ebbbf658 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 29 Aug 2025 12:37:44 +0200 Subject: [PATCH 264/374] get_marginals: exit early if no peaks found to avoid spurious overlap mask --- src/eynollah/utils/marginals.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/eynollah/utils/marginals.py b/src/eynollah/utils/marginals.py index a29e50d..22ada4e 100644 --- a/src/eynollah/utils/marginals.py +++ b/src/eynollah/utils/marginals.py @@ -94,6 +94,8 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve except: point_left=first_nonzero + if point_left == first_nonzero and point_right == last_nonzero: + return text_regions if point_right>=mask_marginals.shape[1]: From 9b5182c1c07ebbdb65ea81978f9c667917b82743 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 26 Aug 2025 21:00:33 +0200 Subject: [PATCH 265/374] utils: introduce box2rect and box2slice --- src/eynollah/utils/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index c479744..bbf30a8 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -300,9 +300,17 @@ def return_x_start_end_mothers_childs_and_type_of_reading_order( x_end_with_child_without_mother, new_main_sep_y) +def box2rect(box: Tuple[int, int, int, int]) -> Tuple[int, int, int, int]: + return (box[1], box[1] + box[3], + box[0], box[0] + box[2]) + +def box2slice(box: Tuple[int, int, int, int]) -> Tuple[slice, slice]: + return (slice(box[1], box[1] + box[3]), + slice(box[0], box[0] + box[2])) + def crop_image_inside_box(box, img_org_copy): - image_box = img_org_copy[box[1] : box[1] + box[3], box[0] : box[0] + box[2]] - return image_box, [box[1], box[1] + box[3], box[0], box[0] + box[2]] + image_box = img_org_copy[box2slice(box)] + return image_box, box2rect(box) def otsu_copy_binary(img): img_r = np.zeros((img.shape[0], img.shape[1], 3)) From 5bff2d156ab32b72470b547870874da3053a3d7b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 26 Aug 2025 21:02:43 +0200 Subject: [PATCH 266/374] use box2rect instead of crop_image_inside_box when no image needed --- src/eynollah/eynollah.py | 8 +++++--- src/eynollah/utils/separate_lines.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index b636b09..6847c1f 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -98,6 +98,8 @@ from .utils.resize import resize_image from .utils import ( boosting_headers_by_longshot_region_segmentation, crop_image_inside_box, + box2rect, + box2slice, find_num_col, otsu_copy_binary, put_drop_out_from_only_drop_model, @@ -1542,7 +1544,7 @@ class Eynollah: all_found_textline_polygons.append(textlines_ins[::-1]) slopes.append(slope_deskew) - _, crop_coor = crop_image_inside_box(boxes[index],image_page_rotated) + crop_coor = box2rect(boxes[index]) all_box_coord.append(crop_coor) return all_found_textline_polygons, boxes, contours, contours_par, all_box_coord, np.array(range(len(contours_par))), slopes @@ -1754,7 +1756,7 @@ class Eynollah: ##polygons_of_images_fin.append(ploy_img_ind) box = cv2.boundingRect(ploy_img_ind) - _, page_coord_img = crop_image_inside_box(box, text_regions_p_true) + page_coord_img = box2rect(box) # cont_page.append(np.array([[page_coord[2], page_coord[0]], # [page_coord[3], page_coord[0]], # [page_coord[3], page_coord[1]], @@ -1768,7 +1770,7 @@ class Eynollah: if h < 150 or w < 150: pass else: - _, page_coord_img = crop_image_inside_box(box, text_regions_p_true) + page_coord_img = box2rect(box) # cont_page.append(np.array([[page_coord[2], page_coord[0]], # [page_coord[3], page_coord[0]], # [page_coord[3], page_coord[1]], diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index ffbfff7..b1a90b5 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -18,6 +18,8 @@ from .contour import ( from . import ( find_num_col_deskew, crop_image_inside_box, + box2rect, + box2slice, ) def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): @@ -1540,7 +1542,7 @@ def do_work_of_slopes_new( logger.debug('enter do_work_of_slopes_new') x, y, w, h = box_text - _, crop_coor = crop_image_inside_box(box_text, image_page_rotated) + crop_coor = box2rect(box_text) mask_textline = np.zeros(textline_mask_tot_ea.shape) mask_textline = cv2.fillPoly(mask_textline, pts=[contour], color=(1,1,1)) all_text_region_raw = textline_mask_tot_ea * mask_textline @@ -1631,7 +1633,7 @@ def do_work_of_slopes_new_curved( slope_for_all = slope_deskew slope = slope_for_all - _, crop_coor = crop_image_inside_box(box_text, image_page_rotated) + crop_coor = box2rect(box_text) if abs(slope_for_all) < 45: textline_region_in_image = np.zeros(textline_mask_tot_ea.shape) @@ -1685,7 +1687,7 @@ def do_work_of_slopes_new_light( logger.debug('enter do_work_of_slopes_new_light') x, y, w, h = box_text - _, crop_coor = crop_image_inside_box(box_text, image_page_rotated) + crop_coor = box2rect(box_text) mask_textline = np.zeros(textline_mask_tot_ea.shape) mask_textline = cv2.fillPoly(mask_textline, pts=[contour], color=(1,1,1)) all_text_region_raw = textline_mask_tot_ea * mask_textline From 5b16c2fc0066f3e1542dfdf7a1fe9f9241401c38 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 26 Aug 2025 21:05:40 +0200 Subject: [PATCH 267/374] avoid pulling unused 'image_page_rotated' through functions --- src/eynollah/eynollah.py | 48 +++++++++++++--------------- src/eynollah/utils/separate_lines.py | 6 ++-- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6847c1f..8f66af5 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1521,7 +1521,7 @@ class Eynollah: self.logger.debug("exit extract_text_regions") return prediction_regions, prediction_regions2 - def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): + def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): polygons_of_textlines = return_contours_of_interested_region(textline_mask_tot,1,0.00001) M_main_tot = [cv2.moments(polygons_of_textlines[j]) @@ -1549,13 +1549,12 @@ class Eynollah: return all_found_textline_polygons, boxes, contours, contours_par, all_box_coord, np.array(range(len(contours_par))), slopes - def get_slopes_and_deskew_new_light(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): + def get_slopes_and_deskew_new_light(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_light") results = self.executor.map(partial(do_work_of_slopes_new_light, textline_mask_tot_ea=textline_mask_tot, - image_page_rotated=image_page_rotated, slope_deskew=slope_deskew,textline_light=self.textline_light, logger=self.logger,), boxes, contours, contours_par, range(len(contours_par))) @@ -1563,13 +1562,12 @@ class Eynollah: self.logger.debug("exit get_slopes_and_deskew_new_light") return tuple(zip(*results)) - def get_slopes_and_deskew_new(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): + def get_slopes_and_deskew_new(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new") results = self.executor.map(partial(do_work_of_slopes_new, textline_mask_tot_ea=textline_mask_tot, - image_page_rotated=image_page_rotated, slope_deskew=slope_deskew, MAX_SLOPE=MAX_SLOPE, KERNEL=KERNEL, @@ -1580,13 +1578,12 @@ class Eynollah: self.logger.debug("exit get_slopes_and_deskew_new") return tuple(zip(*results)) - def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, mask_texts_only, num_col, scale_par, slope_deskew): + def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, boxes, mask_texts_only, num_col, scale_par, slope_deskew): if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_curved") results = self.executor.map(partial(do_work_of_slopes_new_curved, textline_mask_tot_ea=textline_mask_tot, - image_page_rotated=image_page_rotated, mask_texts_only=mask_texts_only, num_col=num_col, scale_par=scale_par, @@ -2935,10 +2932,10 @@ class Eynollah: return slope_deskew def run_marginals( - self, image_page, textline_mask_tot_ea, mask_images, mask_lines, + self, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction): - image_page_rotated, textline_mask_tot = image_page[:, :], textline_mask_tot_ea[:, :] + textline_mask_tot = textline_mask_tot_ea[:, :] textline_mask_tot[mask_images[:, :] == 1] = 0 text_regions_p_1[mask_lines[:, :] == 1] = 3 @@ -2957,10 +2954,7 @@ class Eynollah: except Exception as e: self.logger.error("exception %s", e) - if self.plotter: - self.plotter.save_plot_of_layout_main_all(text_regions_p, image_page) - self.plotter.save_plot_of_layout_main(text_regions_p, image_page) - return textline_mask_tot, text_regions_p, image_page_rotated + return textline_mask_tot, text_regions_p def run_boxes_no_full_layout( self, image_page, textline_mask_tot, text_regions_p, @@ -3112,7 +3106,7 @@ class Eynollah: text_regions_p[:,:][table_prediction[:,:]==1] = 10 img_revised_tab = text_regions_p[:,:] if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - image_page_rotated_n, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ + _, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) @@ -3132,7 +3126,7 @@ class Eynollah: else: if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - image_page_rotated_n, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ + _, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) @@ -4010,9 +4004,12 @@ class Eynollah: text_regions_p_1 = resize_image(text_regions_p_1,img_h_new, img_w_new ) table_prediction = resize_image(table_prediction,img_h_new, img_w_new ) - textline_mask_tot, text_regions_p, image_page_rotated = \ - self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, + textline_mask_tot, text_regions_p = \ + self.run_marginals(textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction) + if self.plotter: + self.plotter.save_plot_of_layout_main_all(text_regions_p, image_page) + self.plotter.save_plot_of_layout_main(text_regions_p, image_page) if self.light_version and num_col_classifier in (1,2): image_page = resize_image(image_page,org_h_l_m, org_w_l_m ) @@ -4021,7 +4018,6 @@ class Eynollah: textline_mask_tot = resize_image(textline_mask_tot,org_h_l_m, org_w_l_m ) text_regions_p_1 = resize_image(text_regions_p_1,org_h_l_m, org_w_l_m ) table_prediction = resize_image(table_prediction,org_h_l_m, org_w_l_m ) - image_page_rotated = resize_image(image_page_rotated,org_h_l_m, org_w_l_m ) self.logger.info("detection of marginals took %.1fs", time.time() - t1) #print("text region early 2 marginal in %.1fs", time.time() - t0) @@ -4197,11 +4193,11 @@ class Eynollah: all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_light2( txt_con_org, contours_only_text_parent, textline_mask_tot_ea_org, - image_page_rotated, boxes_text, slope_deskew) + boxes_text, slope_deskew) all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_light2( polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_org, - image_page_rotated, boxes_marginals, slope_deskew) + boxes_marginals, slope_deskew) #slopes, all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con = \ # self.delete_regions_without_textlines(slopes, all_found_textline_polygons, @@ -4221,11 +4217,11 @@ class Eynollah: all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, \ index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_light( txt_con_org, contours_only_text_parent, textline_mask_tot_ea, - image_page_rotated, boxes_text, slope_deskew) + boxes_text, slope_deskew) all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_light( polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, - image_page_rotated, boxes_marginals, slope_deskew) + boxes_marginals, slope_deskew) #all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( # all_found_textline_polygons, textline_mask_tot_ea_org, type_contour="textline") else: @@ -4233,25 +4229,25 @@ class Eynollah: all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new( txt_con_org, contours_only_text_parent, textline_mask_tot_ea, - image_page_rotated, boxes_text, slope_deskew) + boxes_text, slope_deskew) all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new( polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, - image_page_rotated, boxes_marginals, slope_deskew) + boxes_marginals, slope_deskew) else: scale_param = 1 textline_mask_tot_ea_erode = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=2) all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_curved( txt_con_org, contours_only_text_parent, textline_mask_tot_ea_erode, - image_page_rotated, boxes_text, text_only, + boxes_text, text_only, num_col_classifier, scale_param, slope_deskew) all_found_textline_polygons = small_textlines_to_parent_adherence2( all_found_textline_polygons, textline_mask_tot_ea, num_col_classifier) all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_curved( polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_erode, - image_page_rotated, boxes_marginals, text_only, + boxes_marginals, text_only, num_col_classifier, scale_param, slope_deskew) all_found_textline_polygons_marginals = small_textlines_to_parent_adherence2( all_found_textline_polygons_marginals, textline_mask_tot_ea, num_col_classifier) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index b1a90b5..dcddc65 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1532,7 +1532,7 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map def do_work_of_slopes_new( box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, image_page_rotated, slope_deskew, + textline_mask_tot_ea, slope_deskew, logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None ): if KERNEL is None: @@ -1590,7 +1590,7 @@ def do_work_of_slopes_new( def do_work_of_slopes_new_curved( box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, image_page_rotated, mask_texts_only, num_col, scale_par, slope_deskew, + textline_mask_tot_ea, mask_texts_only, num_col, scale_par, slope_deskew, logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None ): if KERNEL is None: @@ -1679,7 +1679,7 @@ def do_work_of_slopes_new_curved( def do_work_of_slopes_new_light( box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, image_page_rotated, slope_deskew, textline_light, + textline_mask_tot_ea, slope_deskew, textline_light, logger=None ): if logger is None: From 4337d6298596b1272c35b909a0ec0ee50adc4ba2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 26 Aug 2025 21:06:36 +0200 Subject: [PATCH 268/374] =?UTF-8?q?contours:=20rename=20'pixel'=20?= =?UTF-8?q?=E2=86=92=20'label'=20for=20clarity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/eynollah/utils/contour.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 2cd7080..0700ed4 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -141,12 +141,12 @@ def return_parent_contours(contours, hierarchy): if hierarchy[0][i][3] == -1] return contours_parent -def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): +def return_contours_of_interested_region(region_pre_p, label, min_area=0.0002): # pixels of images are identified by 5 if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = (region_pre_p[:, :] == label) * 1 cnts_images = cnts_images.astype(np.uint8) cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) @@ -267,12 +267,12 @@ def get_textregion_contours_in_org_image_light(cnts, img, confidence_matrix): confs.append(np.sum(confidence_matrix * cnt_mask) / np.sum(cnt_mask)) return cnts, confs -def return_contours_of_interested_textline(region_pre_p, pixel): +def return_contours_of_interested_textline(region_pre_p, label): # pixels of images are identified by 5 if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = (region_pre_p[:, :] == label) * 1 cnts_images = cnts_images.astype(np.uint8) cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) @@ -295,12 +295,12 @@ def return_contours_of_image(image): contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) return contours, hierarchy -def return_contours_of_interested_region_by_min_size(region_pre_p, pixel, min_size=0.00003): +def return_contours_of_interested_region_by_min_size(region_pre_p, label, min_size=0.00003): # pixels of images are identified by 5 if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = (region_pre_p[:, :] == label) * 1 cnts_images = cnts_images.astype(np.uint8) cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) @@ -313,12 +313,12 @@ def return_contours_of_interested_region_by_min_size(region_pre_p, pixel, min_si return contours_imgs -def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, max_area): +def return_contours_of_interested_region_by_size(region_pre_p, label, min_area, max_area): # pixels of images are identified by 5 if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = (region_pre_p[:, :] == label) * 1 cnts_images = cnts_images.astype(np.uint8) cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) From f458e3ece01aa7142c77b930dbdf1843c6835d85 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 26 Aug 2025 21:07:18 +0200 Subject: [PATCH 269/374] writer: SeparatorRegion needs SeparatorRegionType (not ImageRegionType) --- src/eynollah/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 92e353f..01c86de 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -296,7 +296,7 @@ class EynollahXmlWriter(): page.add_ImageRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_img[mm], page_coord)))) for mm in range(len(polygons_lines_to_be_written_in_xml)): - page.add_SeparatorRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(polygons_lines_to_be_written_in_xml[mm], [0 , 0, 0, 0])))) + page.add_SeparatorRegion(SeparatorRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(polygons_lines_to_be_written_in_xml[mm], [0 , 0, 0, 0])))) for mm in range(len(found_polygons_tables)): page.add_TableRegion(TableRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_tables[mm], page_coord)))) From dc0caad512219a2e08da3841c215167eed1526bb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 26 Aug 2025 21:07:50 +0200 Subject: [PATCH 270/374] writer: use @type='heading' instead of 'header' --- src/eynollah/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 01c86de..b9e906a 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -268,7 +268,7 @@ class EynollahXmlWriter(): self.logger.debug('len(found_polygons_text_region_h) %s', len(found_polygons_text_region_h)) for mm in range(len(found_polygons_text_region_h)): - textregion = TextRegionType(id=counter.next_region_id, type_='header', + textregion = TextRegionType(id=counter.next_region_id, type_='heading', Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_h[mm], page_coord))) page.add_TextRegion(textregion) From abf5c0f845255f247ce4991d18a5b3b8a3808f4e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 2 Sep 2025 15:01:52 +0200 Subject: [PATCH 271/374] get_smallest_skew: when shifting search range of rotation angle, compare resulting (maximum) variances instead of blindly assuming the new range is better --- src/eynollah/utils/separate_lines.py | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index dcddc65..3363367 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1486,33 +1486,36 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]: angles = np.array([-45, 0, 45, 90,]) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + angle, _ = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + angle, _ = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) elif main_page: angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + angle, var = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) early_slope_edge=11 if abs(angle) > early_slope_edge: if angle < 0: - angles = np.linspace(-90, -12, n_tot_angles) + angles2 = np.linspace(-90, -12, n_tot_angles) else: - angles = np.linspace(90, 12, n_tot_angles) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + angles2 = np.linspace(90, 12, n_tot_angles) + angle2, var2 = get_smallest_skew(img_resized, sigma_des, angles2, map=map, logger=logger, plotter=plotter) + if var2 > var: + angle = angle2 else: angles = np.linspace(-25, 25, int(0.5 * n_tot_angles) + 10) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + angle, var = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) early_slope_edge=22 if abs(angle) > early_slope_edge: if angle < 0: - angles = np.linspace(-90, -25, int(0.5 * n_tot_angles) + 10) + angles2 = np.linspace(-90, -25, int(0.5 * n_tot_angles) + 10) else: - angles = np.linspace(90, 25, int(0.5 * n_tot_angles) + 10) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) - + angles2 = np.linspace(90, 25, int(0.5 * n_tot_angles) + 10) + angle2, var2 = get_smallest_skew(img_resized, sigma_des, angles2, map=map, logger=logger, plotter=plotter) + if var2 > var: + angle = angle2 return angle def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map): @@ -1524,11 +1527,14 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map try: var_res = np.array(results) assert var_res.any() - angle = angles[np.argmax(var_res)] + idx = np.argmax(var_res) + angle = angles[idx] + var = var_res[idx] except: logger.exception("cannot determine best angle among %s", str(angles)) angle = 0 - return angle + var = 0 + return angle, var def do_work_of_slopes_new( box_text, contour, contour_par, index_r_con, From 8be2c7977101080856e4d6e43660a0de055b86c9 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 3 Sep 2025 09:01:18 +0200 Subject: [PATCH 272/374] Revert "deskewing with faster multiprocessing" This reverts commit 5db3e9fa64d39c128bd9bee27c9d0fb73b3459d2. --- src/eynollah/eynollah.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 8f66af5..b450b17 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2926,6 +2926,7 @@ class Eynollah: #print(textline_mask_tot_ea.shape, 'textline_mask_tot_ea deskew') slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True, map=self.executor.map, logger=self.logger, plotter=self.plotter) + if self.plotter: self.plotter.save_deskewed_image(slope_deskew) self.logger.info("slope_deskew: %.2f°", slope_deskew) From 31f240c3b8a6eaa034b5ae02cf009930e8275725 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 2 Sep 2025 15:04:04 +0200 Subject: [PATCH 273/374] do_image_rotation, do_work_of_slopes_new_curved: pass arrays via shared memory --- src/eynollah/eynollah.py | 12 +++++--- src/eynollah/utils/separate_lines.py | 12 ++++++-- src/eynollah/utils/shm.py | 45 ++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 src/eynollah/utils/shm.py diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index b450b17..42af8e4 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -95,6 +95,7 @@ from .utils.drop_capitals import ( ) from .utils.marginals import get_marginals from .utils.resize import resize_image +from .utils.shm import share_ndarray from .utils import ( boosting_headers_by_longshot_region_segmentation, crop_image_inside_box, @@ -1582,9 +1583,11 @@ class Eynollah: if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_curved") - results = self.executor.map(partial(do_work_of_slopes_new_curved, - textline_mask_tot_ea=textline_mask_tot, - mask_texts_only=mask_texts_only, + with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: + with share_ndarray(mask_texts_only) as mask_texts_only_shared: + results = self.executor.map(partial(do_work_of_slopes_new_curved, + textline_mask_tot_ea=textline_mask_tot_shared, + mask_texts_only=mask_texts_only_shared, num_col=num_col, scale_par=scale_par, slope_deskew=slope_deskew, @@ -1593,7 +1596,8 @@ class Eynollah: logger=self.logger, plotter=self.plotter,), boxes, contours, contours_par, range(len(contours_par))) - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) + #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) + results = list(results) # exhaust prior to release self.logger.debug("exit get_slopes_and_deskew_new_curved") return tuple(zip(*results)) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 3363367..e4bb953 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -15,6 +15,7 @@ from .contour import ( return_contours_of_interested_textline, find_contours_mean_y_diff, ) +from .shm import share_ndarray, wrap_ndarray_shared from . import ( find_num_col_deskew, crop_image_inside_box, @@ -1454,7 +1455,8 @@ def separate_lines_new2(img_crop, thetha, num_col, slope_region, logger=None, pl return img_patch_interest_revised -def do_image_rotation(angle, img, sigma_des, logger=None): +@wrap_ndarray_shared(kw='img') +def do_image_rotation(angle, img=None, sigma_des=1.0, logger=None): if logger is None: logger = getLogger(__package__) img_rot = rotate_image(img, angle) @@ -1521,7 +1523,8 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map): if logger is None: logger = getLogger(__package__) - results = list(map(partial(do_image_rotation, img=img, sigma_des=sigma_des, logger=logger), angles)) + with share_ndarray(img) as img_shared: + results = list(map(partial(do_image_rotation, img=img_shared, sigma_des=sigma_des, logger=logger), angles)) if plotter: plotter.save_plot_of_rotation_angle(angles, results) try: @@ -1594,9 +1597,12 @@ def do_work_of_slopes_new( return cnt_clean_rot, box_text, contour, contour_par, crop_coor, index_r_con, slope +@wrap_ndarray_shared(kw='textline_mask_tot_ea') +@wrap_ndarray_shared(kw='mask_texts_only') def do_work_of_slopes_new_curved( box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, mask_texts_only, num_col, scale_par, slope_deskew, + textline_mask_tot_ea=None, mask_texts_only=None, + num_col=1, scale_par=1.0, slope_deskew=0.0, logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None ): if KERNEL is None: diff --git a/src/eynollah/utils/shm.py b/src/eynollah/utils/shm.py new file mode 100644 index 0000000..4b51053 --- /dev/null +++ b/src/eynollah/utils/shm.py @@ -0,0 +1,45 @@ +from multiprocessing import shared_memory +from contextlib import contextmanager +from functools import wraps +import numpy as np + +@contextmanager +def share_ndarray(array: np.ndarray): + size = np.dtype(array.dtype).itemsize * np.prod(array.shape) + shm = shared_memory.SharedMemory(create=True, size=size) + try: + shared_array = np.ndarray(array.shape, dtype=array.dtype, buffer=shm.buf) + shared_array[:] = array[:] + shared_array.flags["WRITEABLE"] = False + yield dict(shape=array.shape, dtype=array.dtype, name=shm.name) + finally: + shm.close() + shm.unlink() + +@contextmanager +def ndarray_shared(array: dict): + shm = shared_memory.SharedMemory(name=array['name']) + try: + array = np.ndarray(array['shape'], dtype=array['dtype'], buffer=shm.buf) + yield array + finally: + shm.close() + +def wrap_ndarray_shared(kw=None): + def wrapper(f): + if kw is None: + @wraps(f) + def shared_func(array, *args, **kwargs): + with ndarray_shared(array) as ndarray: + return f(ndarray, *args, **kwargs) + return shared_func + else: + @wraps(f) + def shared_func(*args, **kwargs): + array = kwargs.pop(kw) + with ndarray_shared(array) as ndarray: + kwargs[kw] = ndarray + return f(*args, **kwargs) + return shared_func + return wrapper + From 0662ece536e090989ad4e2281317336129eae468 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 4 Sep 2025 15:18:55 +0200 Subject: [PATCH 274/374] do_work_of_slopes*: use shm also in non-light mode(s) --- src/eynollah/eynollah.py | 33 ++++++++++++++++------------ src/eynollah/utils/separate_lines.py | 6 +++-- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 42af8e4..6333ca5 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1554,11 +1554,14 @@ class Eynollah: if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_light") - results = self.executor.map(partial(do_work_of_slopes_new_light, - textline_mask_tot_ea=textline_mask_tot, - slope_deskew=slope_deskew,textline_light=self.textline_light, - logger=self.logger,), - boxes, contours, contours_par, range(len(contours_par))) + with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: + results = self.executor.map(partial(do_work_of_slopes_new_light, + textline_mask_tot_ea=textline_mask_tot_shared, + slope_deskew=slope_deskew, + textline_light=self.textline_light, + logger=self.logger,), + boxes, contours, contours_par, range(len(contours_par))) + results = list(results) # exhaust prior to release #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) self.logger.debug("exit get_slopes_and_deskew_new_light") return tuple(zip(*results)) @@ -1567,14 +1570,16 @@ class Eynollah: if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new") - results = self.executor.map(partial(do_work_of_slopes_new, - textline_mask_tot_ea=textline_mask_tot, - slope_deskew=slope_deskew, - MAX_SLOPE=MAX_SLOPE, - KERNEL=KERNEL, - logger=self.logger, - plotter=self.plotter,), - boxes, contours, contours_par, range(len(contours_par))) + with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: + results = self.executor.map(partial(do_work_of_slopes_new, + textline_mask_tot_ea=textline_mask_tot_shared, + slope_deskew=slope_deskew, + MAX_SLOPE=MAX_SLOPE, + KERNEL=KERNEL, + logger=self.logger, + plotter=self.plotter,), + boxes, contours, contours_par, range(len(contours_par))) + results = list(results) # exhaust prior to release #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) self.logger.debug("exit get_slopes_and_deskew_new") return tuple(zip(*results)) @@ -1596,8 +1601,8 @@ class Eynollah: logger=self.logger, plotter=self.plotter,), boxes, contours, contours_par, range(len(contours_par))) - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) results = list(results) # exhaust prior to release + #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) self.logger.debug("exit get_slopes_and_deskew_new_curved") return tuple(zip(*results)) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index e4bb953..1a2f511 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1539,9 +1539,10 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map var = 0 return angle, var +@wrap_ndarray_shared(kw='textline_mask_tot_ea') def do_work_of_slopes_new( box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, slope_deskew, + textline_mask_tot_ea=None, slope_deskew=0.0, logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None ): if KERNEL is None: @@ -1689,9 +1690,10 @@ def do_work_of_slopes_new_curved( return textlines_cnt_per_region[::-1], box_text, contour, contour_par, crop_coor, index_r_con, slope +@wrap_ndarray_shared(kw='textline_mask_tot_ea') def do_work_of_slopes_new_light( box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, slope_deskew, textline_light, + textline_mask_tot_ea=None, slope_deskew=0, textline_light=True, logger=None ): if logger is None: From 04c3d7dd1b98b01adf2b8ccd72830ad5fd9a4e95 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 18 Sep 2025 20:07:54 +0200 Subject: [PATCH 275/374] get_smallest_skew: avoid shm if no ProcessPoolExecutor is passed --- src/eynollah/utils/separate_lines.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 1a2f511..4d8badb 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1469,7 +1469,7 @@ def do_image_rotation(angle, img=None, sigma_des=1.0, logger=None): return var def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, - main_page=False, logger=None, plotter=None, map=map): + main_page=False, logger=None, plotter=None, map=None): if main_page and plotter: plotter.save_plot_of_textline_density(img_patch_org) @@ -1523,8 +1523,13 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map): if logger is None: logger = getLogger(__package__) - with share_ndarray(img) as img_shared: - results = list(map(partial(do_image_rotation, img=img_shared, sigma_des=sigma_des, logger=logger), angles)) + if map is None: + results = [do_image_rotation.__wrapped__(angle, img=img, sigma_des=sigma_des, logger=logger) + for angle in angles] + else: + with share_ndarray(img) as img_shared: + results = list(map(partial(do_image_rotation, img=img_shared, sigma_des=sigma_des, logger=logger), + angles)) if plotter: plotter.save_plot_of_rotation_angle(angles, results) try: From b94c96fcbbb5bbce72bc9cdc9b334953abd774ad Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 20 Sep 2025 00:56:33 +0200 Subject: [PATCH 276/374] find_num_col: exit early if empty (avoiding exceptions) --- src/eynollah/utils/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index bbf30a8..9daec7d 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -383,6 +383,10 @@ def find_num_col_deskew(regions_without_separators, sigma_, multiplier=3.8): return np.std(z) def find_num_col(regions_without_separators, num_col_classifier, tables, multiplier=3.8): + if not regions_without_separators.any(): + return 0, [] + #plt.imshow(regions_without_separators) + #plt.show() regions_without_separators_0 = regions_without_separators.sum(axis=0) ##plt.plot(regions_without_separators_0) ##plt.show() @@ -402,6 +406,9 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl zneg = gaussian_filter1d(zneg, sigma_) peaks_neg, _ = find_peaks(zneg, height=0) + #plt.plot(zneg) + #plt.plot(peaks_neg, zneg[peaks_neg], 'rx') + #plt.show() peaks, _ = find_peaks(z, height=0) peaks_neg = peaks_neg - 10 - 10 @@ -416,9 +423,13 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl (peaks_neg < (regions_without_separators.shape[1] - 370))] interest_pos = z[peaks] interest_pos = interest_pos[interest_pos > 10] + if not interest_pos.any(): + return 0, [] # plt.plot(z) # plt.show() interest_neg = z[peaks_neg] + if not interest_neg.any(): + return 0, [] min_peaks_pos = np.min(interest_pos) max_peaks_pos = np.max(interest_pos) From 0366707136568241c42bac2f3bf675dda5989fe2 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sat, 20 Sep 2025 00:57:00 +0200 Subject: [PATCH 277/374] get_smallest_skew: do not pass logger --- src/eynollah/utils/separate_lines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 4d8badb..1d27a17 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1528,7 +1528,7 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map for angle in angles] else: with share_ndarray(img) as img_shared: - results = list(map(partial(do_image_rotation, img=img_shared, sigma_des=sigma_des, logger=logger), + results = list(map(partial(do_image_rotation, img=img_shared, sigma_des=sigma_des, logger=None), angles)) if plotter: plotter.save_plot_of_rotation_angle(angles, results) From 758602403eb92625608d04e7d77fcbf896c55e2d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sun, 21 Sep 2025 21:35:22 +0200 Subject: [PATCH 278/374] replace loky with concurrent.futures.ProcessPoolExecutor (faster) --- src/eynollah/eynollah.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6333ca5..1c70498 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -23,7 +23,7 @@ import gc import copy import json -from loky import ProcessPoolExecutor +from concurrent.futures import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 import numpy as np @@ -244,7 +244,7 @@ class Eynollah: self.num_col_lower = num_col_lower self.logger = logger if logger else getLogger('eynollah') # for parallelization of CPU-intensive tasks: - self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) + self.executor = ProcessPoolExecutor(max_workers=cpu_count()) atexit.register(self.executor.shutdown) self.dir_models = dir_models self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" From 53c1ca11fc57c86276fc307fff01050f78517e24 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Mon, 29 Sep 2025 22:15:17 +0200 Subject: [PATCH 279/374] Update README.md --- README.md | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e576f4d..9dc4824 100644 --- a/README.md +++ b/README.md @@ -11,23 +11,24 @@ ![](https://user-images.githubusercontent.com/952378/102350683-8a74db80-3fa5-11eb-8c7e-f743f7d6eae2.jpg) ## Features -* Support for up to 10 segmentation classes: +* Support for 10 distinct segmentation classes: * background, [page border](https://ocr-d.de/en/gt-guidelines/trans/lyRand.html), [text region](https://ocr-d.de/en/gt-guidelines/trans/lytextregion.html#textregionen__textregion_), [text line](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextLineType.html), [header](https://ocr-d.de/en/gt-guidelines/trans/lyUeberschrift.html), [image](https://ocr-d.de/en/gt-guidelines/trans/lyBildbereiche.html), [separator](https://ocr-d.de/en/gt-guidelines/trans/lySeparatoren.html), [marginalia](https://ocr-d.de/en/gt-guidelines/trans/lyMarginalie.html), [initial](https://ocr-d.de/en/gt-guidelines/trans/lyInitiale.html), [table](https://ocr-d.de/en/gt-guidelines/trans/lyTabellen.html) * Support for various image optimization operations: * cropping (border detection), binarization, deskewing, dewarping, scaling, enhancing, resizing -* Text line segmentation to bounding boxes or polygons (contours) including for curved lines and vertical text -* Detection of reading order (left-to-right or right-to-left) +* Textline segmentation to bounding boxes or polygons (contours) including for curved lines and vertical text +* Text recognition (OCR) using either CNN-RNN or Transformer models +* Detection of reading order (left-to-right or right-to-left) using either heuristics or trainable models * Output in [PAGE-XML](https://github.com/PRImA-Research-Lab/PAGE-XML) * [OCR-D](https://github.com/qurator-spk/eynollah#use-as-ocr-d-processor) interface -:warning: Development is currently focused on achieving the best possible quality of results for a wide variety of -historical documents and therefore processing can be very slow. We aim to improve this, but contributions are welcome. +:warning: Development is focused on achieving the best quality of results for a wide variety of historical +documents and therefore processing can be very slow. We aim to improve this, but contributions are welcome. ## Installation Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported. -For (limited) GPU support the CUDA toolkit needs to be installed. +For (limited) GPU support the CUDA toolkit needs to be installed. A known working config is CUDA `11` with cuDNN `8.6`. You can either install from PyPI @@ -56,26 +57,27 @@ make install EXTRAS=OCR Pretrained models can be downloaded from [zenodo](https://zenodo.org/records/17194824) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). -For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). +For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). +Model cards are also provided for our trained models. ## Training -In case you want to train your own model with Eynollah, have see the +In case you want to train your own model with Eynollah, see the documentation in [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md) and use the tools in the [`train` folder](https://github.com/qurator-spk/eynollah/tree/main/train). ## Usage Eynollah supports five use cases: layout analysis (segmentation), binarization, -image enhancement, text recognition (OCR), and (trainable) reading order detection. +image enhancement, text recognition (OCR), and reading order detection. ### Layout Analysis -The layout analysis module is responsible for detecting layouts, identifying text lines, and determining reading order -using both heuristic methods or a machine-based reading order detection model. +The layout analysis module is responsible for detecting layout elements, identifying text lines, and determining reading +order using either heuristic methods or a reading order detection model. -Note that there are currently two supported ways for reading order detection: either as part of layout analysis based -on image input, or, currently under development, for given layout analysis results based on PAGE-XML data as input. +Reading order detection can be performed either as part of layout analysis based on image input, or, currently under +development, based on pre-existing layout analysis results in PAGE-XML format as input. The command-line interface for layout analysis can be called like this: @@ -108,15 +110,15 @@ The following options can be used to further configure the processing: | `-sp ` | save cropped page image to this directory | | `-sa ` | save all (plot, enhanced/binary image, layout) to this directory | -If no option is set, the tool performs layout detection of main regions (background, text, images, separators +If no further option is set, the tool performs layout detection of main regions (background, text, images, separators and marginals). -The best output quality is produced when RGB images are used as input rather than greyscale or binarized images. +The best output quality is achieved when RGB images are used as input rather than greyscale or binarized images. ### Binarization The binarization module performs document image binarization using pretrained pixelwise segmentation models. -The command-line interface for binarization of single image can be called like this: +The command-line interface for binarization can be called like this: ```sh eynollah binarization \ @@ -127,16 +129,16 @@ eynollah binarization \ ### OCR -The OCR module performs text recognition from images using two main families of pretrained models: CNN-RNN–based OCR and Transformer-based OCR. +The OCR module performs text recognition using either a CNN-RNN model or a Transformer model. -The command-line interface for ocr can be called like this: +The command-line interface for OCR can be called like this: ```sh eynollah ocr \ -i | -di \ -dx \ -o \ - -m | --model_name \ + -m | --model_name \ ``` ### Machine-based-reading-order @@ -174,20 +176,18 @@ If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynol ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_5_0 -Still, in general, it makes more sense to add other workflow steps **after** Eynollah. +In general, it makes more sense to add other workflow steps **after** Eynollah. -There is also an OCR-D processor for the binarization: +There is also an OCR-D processor for binarization: ocrd-sbb-binarize -I OCR-D-IMG -O OCR-D-BIN -P models default-2021-03-09 #### Additional documentation -Please check the [wiki](https://github.com/qurator-spk/eynollah/wiki). +Additional documentation is available in the [docs](https://github.com/qurator-spk/eynollah/tree/main/docs) directory. ## How to cite -If you find this tool useful in your work, please consider citing our paper: - ```bibtex @inproceedings{hip23rezanezhad, title = {Document Layout Analysis with Deep Learning and Heuristics}, From 070dafca759f568a7d4bfa6ddfd9cb62324c87f3 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Mon, 29 Sep 2025 22:17:27 +0200 Subject: [PATCH 280/374] remove duplicate LICENSE --- train/LICENSE | 201 -------------------------------------------------- 1 file changed, 201 deletions(-) delete mode 100644 train/LICENSE diff --git a/train/LICENSE b/train/LICENSE deleted file mode 100644 index 261eeb9..0000000 --- a/train/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. From c0137c29ad46adf2096664632e9a20a30afbfe09 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 02:23:43 +0200 Subject: [PATCH 281/374] try to fix the failed outsourcing of utils_ocr --- src/eynollah/eynollah.py | 63 ++------------------------------- src/eynollah/utils/utils_ocr.py | 1 + 2 files changed, 3 insertions(+), 61 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 32490a2..192f6f4 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3917,34 +3917,6 @@ class Eynollah: region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] return ordered, region_ids - def return_list_of_contours_with_desired_order(self, ls_cons, sorted_indexes): - return [ls_cons[sorted_indexes[index]] for index in range(len(sorted_indexes))] - - def return_it_in_two_groups(self, x_differential): - split = [ind if x_differential[ind]!=x_differential[ind+1] else -1 - for ind in range(len(x_differential)-1)] - split_masked = list( np.array(split[:])[np.array(split[:])!=-1] ) - if 0 not in split_masked: - split_masked.insert(0, -1) - split_masked.append(len(x_differential)-1) - - split_masked = np.array(split_masked) +1 - - sums = [np.sum(x_differential[split_masked[ind]:split_masked[ind+1]]) - for ind in range(len(split_masked)-1)] - - indexes_to_bec_changed = [ind if (np.abs(sums[ind-1]) > np.abs(sums[ind]) and - np.abs(sums[ind+1]) > np.abs(sums[ind])) else -1 - for ind in range(1,len(sums)-1)] - indexes_to_bec_changed_filtered = np.array(indexes_to_bec_changed)[np.array(indexes_to_bec_changed)!=-1] - - x_differential_new = np.copy(x_differential) - for i in indexes_to_bec_changed_filtered: - i_slice = slice(split_masked[i], split_masked[i+1]) - x_differential_new[i_slice] = -1 * np.array(x_differential)[i_slice] - - return x_differential_new - def return_start_and_end_of_common_text_of_textline_ocr(self,textline_image, ind_tot): width = np.shape(textline_image)[1] height = np.shape(textline_image)[0] @@ -3988,36 +3960,6 @@ class Eynollah: else: pass - def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.06*width) - - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: - #print(len(peaks_real), 'len(peaks_real)') - - peaks_real = peaks_real[(peaks_realwidth1)] - - arg_max = np.argmax(sum_smoothed[peaks_real]) - peaks_final = peaks_real[arg_max] - - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peaks_final, peaks_final], [0, height-1]) - ##plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') - - return peaks_final - else: - return None - def return_start_and_end_of_common_text_of_textline_ocr_new_splitted( self, peaks_real, sum_smoothed, start_split, end_split): @@ -4079,8 +4021,7 @@ class Eynollah: #width1 = int ( width/2. - common_window ) #width2 = int ( width/2. + common_window ) - split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section( - textline_image, ind_tot) + split_point = return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image) if split_point: image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) @@ -5144,7 +5085,7 @@ class Eynollah: box_ind = all_box_coord[indexing] #print(ind_poly,np.shape(ind_poly), 'ind_poly') #print(box_ind) - ind_poly = self.return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) + ind_poly = return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) #print(ind_poly_copy) ind_poly[ind_poly<0] = 0 x, y, w, h = cv2.boundingRect(ind_poly) diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 4fa99f7..5f19387 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -92,6 +92,7 @@ def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(t return peaks_final else: return None + # Function to fit text inside the given area def fit_text_single_line(draw, text, font_path, max_width, max_height): initial_font_size = 50 From f857ee7b518e23c62b28aab32cd64d396da836fe Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 19 Sep 2025 02:12:18 +0200 Subject: [PATCH 282/374] simplify --- src/eynollah/eynollah.py | 23 +++-------------------- src/eynollah/utils/__init__.py | 2 +- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 192f6f4..0c9692e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -3182,26 +3182,9 @@ class Eynollah: num_col = num_col + 1 if not num_column_is_classified: num_col_classifier = num_col + 1 - if self.num_col_upper and self.num_col_lower: - if self.num_col_upper == self.num_col_lower: - num_col_classifier = self.num_col_upper - else: - if num_col_classifier < self.num_col_lower: - num_col_classifier = self.num_col_lower - if num_col_classifier > self.num_col_upper: - num_col_classifier = self.num_col_upper - - elif self.num_col_lower and not self.num_col_upper: - if num_col_classifier < self.num_col_lower: - num_col_classifier = self.num_col_lower - - elif self.num_col_upper and not self.num_col_lower: - if num_col_classifier > self.num_col_upper: - num_col_classifier = self.num_col_upper - - else: - pass - + num_col_classifier = min(self.num_col_upper or num_col_classifier, + max(self.num_col_lower or num_col_classifier, + num_col_classifier)) except Exception as why: self.logger.error(why) num_col = None diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 243430e..f8926cf 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1675,9 +1675,9 @@ def return_boxes_of_images_by_order_of_reading_new( peaks_neg_fin=[] num_col = 0 try: - peaks_neg_fin_org=np.copy(peaks_neg_fin) if (len(peaks_neg_fin)+1) Date: Tue, 30 Sep 2025 03:52:19 +0200 Subject: [PATCH 283/374] indent extremely long lines --- src/eynollah/eynollah.py | 750 ++++++++++++++++++--------- src/eynollah/utils/__init__.py | 30 +- src/eynollah/utils/separate_lines.py | 136 +++-- src/eynollah/utils/utils_ocr.py | 25 +- 4 files changed, 652 insertions(+), 289 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0c9692e..2e31433 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -272,7 +272,6 @@ class Eynollah: else: self.threshold_art_class_textline = 0.1 - self.dir_models = dir_models self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" self.model_dir_of_col_classifier = dir_models + "/eynollah-column-classifier_20210425" @@ -289,8 +288,17 @@ class Eynollah: self.model_page_dir = dir_models + "/model_eynollah_page_extraction_20250915" self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" - self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_eynollah_reading_order_20250824"#"/model_mb_ro_aug_ens_11"#"/model_step_3200000_mb_ro"#"/model_ens_reading_order_machine_based"#"/model_mb_ro_aug_ens_8"#"/model_ens_reading_order_machine_based" + self.model_region_dir_p_ens_light_only_images_extraction = (dir_models + + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" + ) + self.model_reading_order_dir = (dir_models + + "/model_eynollah_reading_order_20250824" + #"/model_mb_ro_aug_ens_11" + #"/model_step_3200000_mb_ro" + #"/model_ens_reading_order_machine_based" + #"/model_mb_ro_aug_ens_8" + #"/model_ens_reading_order_machine_based" + ) #"/modelens_12sp_elay_0_3_4__3_6_n" #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" @@ -379,11 +387,9 @@ class Eynollah: self.b_s_ocr = 8 else: self.b_s_ocr = int(batch_size_ocr) - with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: characters = json.load(config_file) - AUTOTUNE = tf.data.AUTOTUNE @@ -840,7 +846,9 @@ class Eynollah: self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False, thresholding_for_fl_light_version=False, threshold_art_class_textline=0.1): + thresholding_for_artificial_class_in_light_version=False, + thresholding_for_fl_light_version=False, + threshold_art_class_textline=0.1): self.logger.debug("enter do_prediction") img_height_model = model.layers[-1].output_shape[1] @@ -1254,7 +1262,9 @@ class Eynollah: self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False, threshold_art_class_textline=0.1, threshold_art_class_layout=0.1): + thresholding_for_artificial_class_in_light_version=False, + threshold_art_class_textline=0.1, + threshold_art_class_layout=0.1): self.logger.debug("enter do_prediction_new_concept") img_height_model = model.layers[-1].output_shape[1] @@ -1384,7 +1394,8 @@ class Eynollah: for i_batch, j_batch in zip(list_i_s, list_j_s): seg_in = seg[indexer_inside_batch] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): seg_in_art = seg_art[indexer_inside_batch] index_y_u_in = list_y_u[indexer_inside_batch] @@ -1404,7 +1415,8 @@ class Eynollah: label_p_pred[0, 0:-margin or None, 0:-margin or None, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + 0:index_x_u_in - margin, 1] = \ seg_in_art[0:-margin or None, @@ -1421,7 +1433,8 @@ class Eynollah: label_p_pred[0, margin:, margin:, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + margin:index_x_u_in - 0, 1] = \ seg_in_art[margin:, @@ -1439,7 +1452,8 @@ class Eynollah: 0:-margin or None, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + 0:index_x_u_in - margin, 1] = \ seg_in_art[margin:, @@ -1456,7 +1470,8 @@ class Eynollah: label_p_pred[0, 0:-margin or None, margin:, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - 0, 1] = \ seg_in_art[0:-margin or None, @@ -1473,7 +1488,8 @@ class Eynollah: label_p_pred[0, margin:-margin or None, 0:-margin or None, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + 0:index_x_u_in - margin, 1] = \ seg_in_art[margin:-margin or None, @@ -1489,7 +1505,8 @@ class Eynollah: label_p_pred[0, margin:-margin or None, margin:, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - 0, 1] = \ seg_in_art[margin:-margin or None, @@ -1505,7 +1522,8 @@ class Eynollah: label_p_pred[0, 0:-margin or None, margin:-margin or None, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + 0:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - margin, 1] = \ seg_in_art[0:-margin or None, @@ -1521,7 +1539,8 @@ class Eynollah: label_p_pred[0, margin:, margin:-margin or None, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + margin:index_y_u_in - 0, index_x_d_in + margin:index_x_u_in - margin, 1] = \ seg_in_art[margin:, @@ -1537,7 +1556,8 @@ class Eynollah: label_p_pred[0, margin:-margin or None, margin:-margin or None, 1] - if thresholding_for_artificial_class_in_light_version or thresholding_for_some_classes_in_light_version: + if (thresholding_for_artificial_class_in_light_version or + thresholding_for_some_classes_in_light_version): prediction_true[index_y_d_in + margin:index_y_u_in - margin, index_x_d_in + margin:index_x_u_in - margin, 1] = \ seg_in_art[margin:-margin or None, @@ -1686,7 +1706,10 @@ class Eynollah: else: img = resize_image(img, int(img_height_h * 2500 / float(img_width_h)), 2500).astype(np.uint8) - prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1, n_batch_inference=3, thresholding_for_fl_light_version=thresholding_for_fl_light_version) + prediction_regions = self.do_prediction(patches, img, model_region, + marginal_of_patch_percent=0.1, + n_batch_inference=3, + thresholding_for_fl_light_version=thresholding_for_fl_light_version) prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) self.logger.debug("exit extract_text_regions") return prediction_regions, prediction_regions @@ -1839,7 +1862,10 @@ class Eynollah: cy_textline_in = [cy_main_tot[ind] for ind in indexes_in] w_h_textlines_in = [w_h_textlines[ind][0] / float(w_h_textlines[ind][1]) for ind in indexes_in] - textlines_ins = self.get_textlines_of_a_textregion_sorted(textlines_ins, cx_textline_in, cy_textline_in, w_h_textlines_in) + textlines_ins = self.get_textlines_of_a_textregion_sorted(textlines_ins, + cx_textline_in, + cy_textline_in, + w_h_textlines_in) all_found_textline_polygons.append(textlines_ins)#[::-1]) slopes.append(slope_deskew) @@ -1847,7 +1873,13 @@ class Eynollah: crop_coor = box2rect(boxes[index]) all_box_coord.append(crop_coor) - return all_found_textline_polygons, boxes, contours, contours_par, all_box_coord, np.array(range(len(contours_par))), slopes + return (all_found_textline_polygons, + boxes, + contours, + contours_par, + all_box_coord, + np.array(range(len(contours_par))), + slopes) def get_slopes_and_deskew_new_light(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): if not len(contours): @@ -1883,7 +1915,8 @@ class Eynollah: self.logger.debug("exit get_slopes_and_deskew_new") return tuple(zip(*results)) - def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, boxes, mask_texts_only, num_col, scale_par, slope_deskew): + def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, boxes, + mask_texts_only, num_col, scale_par, slope_deskew): if not len(contours): return [], [], [], [], [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_curved") @@ -1914,10 +1947,11 @@ class Eynollah: img_w = img_org.shape[1] img = resize_image(img_org, int(img_org.shape[0] * scaler_h), int(img_org.shape[1] * scaler_w)) - prediction_textline = self.do_prediction( - use_patches, img, self.model_textline, - marginal_of_patch_percent=0.15, n_batch_inference=3, - thresholding_for_artificial_class_in_light_version=self.textline_light, threshold_art_class_textline=self.threshold_art_class_textline) + prediction_textline = self.do_prediction(use_patches, img, self.model_textline, + marginal_of_patch_percent=0.15, + n_batch_inference=3, + thresholding_for_artificial_class_in_light_version=self.textline_light, + threshold_art_class_textline=self.threshold_art_class_textline) #if not self.textline_light: #if num_col_classifier==1: #prediction_textline_nopatch = self.do_prediction(False, img, self.model_textline) @@ -2009,12 +2043,14 @@ class Eynollah: boxes_sub_new = [] poly_sub = [] for mv in range(len(boxes_per_process)): - crop_img, _ = crop_image_inside_box(boxes_per_process[mv], np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) + crop_img, _ = crop_image_inside_box(boxes_per_process[mv], + np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) crop_img = crop_img[:, :, 0] crop_img = cv2.erode(crop_img, KERNEL, iterations=2) try: textline_con, hierarchy = return_contours_of_image(crop_img) - textline_con_fil = filter_contours_area_of_image(crop_img, textline_con, hierarchy, max_area=1, min_area=0.0008) + textline_con_fil = filter_contours_area_of_image(crop_img, textline_con, hierarchy, + max_area=1, min_area=0.0008) y_diff_mean = find_contours_mean_y_diff(textline_con_fil) sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) crop_img[crop_img > 0] = 1 @@ -2139,7 +2175,13 @@ class Eynollah: [page_coord_img[2], page_coord_img[1]]])) self.logger.debug("exit get_regions_extract_images_only") - return text_regions_p_true, erosion_hurts, polygons_seplines, polygons_of_images_fin, image_page, page_coord, cont_page + return (text_regions_p_true, + erosion_hurts, + polygons_seplines, + polygons_of_images_fin, + image_page, + page_coord, + cont_page) def get_regions_light_v(self,img,is_image_enhanced, num_col_classifier, skip_layout_and_reading_order=False): self.logger.debug("enter get_regions_light_v") @@ -2197,7 +2239,8 @@ class Eynollah: #print("inside 1 ", time.time()-t_in) ###textline_mask_tot_ea = self.run_textline(img_bin) - self.logger.debug("detecting textlines on %s with %d colors", str(img_resized.shape), len(np.unique(img_resized))) + self.logger.debug("detecting textlines on %s with %d colors", + str(img_resized.shape), len(np.unique(img_resized))) textline_mask_tot_ea = self.run_textline(img_resized, num_col_classifier) textline_mask_tot_ea = resize_image(textline_mask_tot_ea,img_height_h, img_width_h ) @@ -2214,13 +2257,15 @@ class Eynollah: img_resized.shape[1], img_resized.shape[0], num_col_classifier) prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( True, img_resized, self.model_region_1_2, n_batch_inference=1, - thresholding_for_some_classes_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) + thresholding_for_some_classes_in_light_version=True, + threshold_art_class_layout=self.threshold_art_class_layout) else: prediction_regions_org = np.zeros((self.image_org.shape[0], self.image_org.shape[1], 3)) confidence_matrix = np.zeros((self.image_org.shape[0], self.image_org.shape[1])) prediction_regions_page, confidence_matrix_page = self.do_prediction_new_concept( False, self.image_page_org_size, self.model_region_1_2, n_batch_inference=1, - thresholding_for_artificial_class_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) + thresholding_for_artificial_class_in_light_version=True, + threshold_art_class_layout=self.threshold_art_class_layout) ys = slice(*self.page_coord[0:2]) xs = slice(*self.page_coord[2:4]) prediction_regions_org[ys, xs] = prediction_regions_page @@ -2233,8 +2278,11 @@ class Eynollah: img_resized.shape[1], img_resized.shape[0], new_h, num_col_classifier) prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( True, img_resized, self.model_region_1_2, n_batch_inference=2, - thresholding_for_some_classes_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) - ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, n_batch_inference=3, thresholding_for_some_classes_in_light_version=True) + thresholding_for_some_classes_in_light_version=True, + threshold_art_class_layout=self.threshold_art_class_layout) + ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, + ###n_batch_inference=3, + ###thresholding_for_some_classes_in_light_version=True) #print("inside 3 ", time.time()-t_in) #plt.imshow(prediction_regions_org[:,:,0]) #plt.show() @@ -2297,7 +2345,12 @@ class Eynollah: #plt.show() #print("inside 4 ", time.time()-t_in) self.logger.debug("exit get_regions_light_v") - return text_regions_p_true, erosion_hurts, polygons_seplines, textline_mask_tot_ea, img_bin, confidence_matrix + return (text_regions_p_true, + erosion_hurts, + polygons_seplines, + textline_mask_tot_ea, + img_bin, + confidence_matrix) else: img_bin = resize_image(img_bin,img_height_h, img_width_h ) self.logger.debug("exit get_regions_light_v") @@ -2417,14 +2470,10 @@ class Eynollah: #img = resize_image(img_org, int(img_org.shape[0]*1), int(img_org.shape[1]*1)) #prediction_regions_org = self.do_prediction(True, img, self.model_region) - #prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - #prediction_regions_org = prediction_regions_org[:,:,0] - #prediction_regions_org[(prediction_regions_org[:,:] == 1) & (mask_zeros_y[:,:] == 1)]=0 - mask_lines_only = (prediction_regions_org == 3)*1 mask_texts_only = (prediction_regions_org == 1)*1 mask_images_only= (prediction_regions_org == 2)*1 @@ -2843,7 +2892,8 @@ class Eynollah: contours_new.append(contours_sep[ji]) if num_col_classifier>=2: only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1])) - only_recent_contour_image= cv2.fillPoly(only_recent_contour_image, pts=[contours_sep[ji]], color=(1,1,1)) + only_recent_contour_image= cv2.fillPoly(only_recent_contour_image, + pts=[contours_sep[ji]], color=(1,1,1)) table_pixels_masked_from_early_pre = only_recent_contour_image * table_prediction_early iou_in = 100. * table_pixels_masked_from_early_pre.sum() / only_recent_contour_image.sum() #print(iou_in,'iou_in_in1') @@ -2928,9 +2978,11 @@ class Eynollah: contours,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) if indiv==pixel_table: - main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area = 1, min_area = 0.001) + main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, + max_area=1, min_area=0.001) else: - main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area = 1, min_area = min_area) + main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, + max_area=1, min_area=min_area) img_comm = cv2.fillPoly(img_comm, pts = main_contours, color = (indiv, indiv, indiv)) img_comm = img_comm.astype(np.uint8) @@ -2965,8 +3017,14 @@ class Eynollah: y_min_main_line ,y_max_main_line=find_features_of_contours(contours_line) y_min_main_tab ,y_max_main_tab=find_features_of_contours(contours_tab) - cx_tab_m_text,cy_tab_m_text ,x_min_tab_m_text , x_max_tab_m_text, y_min_tab_m_text ,y_max_tab_m_text, _= find_new_features_of_contours(contours_table_m_text) - cx_tabl,cy_tabl ,x_min_tabl , x_max_tabl, y_min_tabl ,y_max_tabl,_= find_new_features_of_contours(contours_tab) + (cx_tab_m_text, cy_tab_m_text, + x_min_tab_m_text, x_max_tab_m_text, + y_min_tab_m_text, y_max_tab_m_text, + _) = find_new_features_of_contours(contours_table_m_text) + (cx_tabl, cy_tabl, + x_min_tabl, x_max_tabl, + y_min_tabl, y_max_tabl, + _) = find_new_features_of_contours(contours_tab) if len(y_min_main_tab )>0: y_down_tabs=[] @@ -2976,9 +3034,15 @@ class Eynollah: y_down_tab=[] y_up_tab=[] for i_l in range(len(y_min_main_line)): - if y_min_main_tab[i_t]>y_min_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l] and y_min_main_tab[i_t]>y_max_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l]: + if (y_min_main_tab[i_t] > y_min_main_line[i_l] and + y_max_main_tab[i_t] > y_min_main_line[i_l] and + y_min_main_tab[i_t] > y_max_main_line[i_l] and + y_max_main_tab[i_t] > y_min_main_line[i_l]): pass - elif y_min_main_tab[i_t]= SLOPE_THRESHOLD: _, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ - rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) + rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, + table_prediction, slope_deskew) - text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) - textline_mask_tot_d = resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1]) - table_prediction_n = resize_image(table_prediction_n,text_regions_p.shape[0],text_regions_p.shape[1]) + text_regions_p_1_n = resize_image(text_regions_p_1_n, + text_regions_p.shape[0], + text_regions_p.shape[1]) + textline_mask_tot_d = resize_image(textline_mask_tot_d, + text_regions_p.shape[0], + text_regions_p.shape[1]) + table_prediction_n = resize_image(table_prediction_n, + text_regions_p.shape[0], + text_regions_p.shape[1]) regions_without_separators_d = (text_regions_p_1_n[:,:] == 1)*1 regions_without_separators_d[table_prediction_n[:,:] == 1] = 1 @@ -3502,11 +3580,18 @@ class Eynollah: else: if np.abs(slope_deskew) >= SLOPE_THRESHOLD: _, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ - rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) + rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, + table_prediction, slope_deskew) - text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) - textline_mask_tot_d = resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1]) - table_prediction_n = resize_image(table_prediction_n,text_regions_p.shape[0],text_regions_p.shape[1]) + text_regions_p_1_n = resize_image(text_regions_p_1_n, + text_regions_p.shape[0], + text_regions_p.shape[1]) + textline_mask_tot_d = resize_image(textline_mask_tot_d, + text_regions_p.shape[0], + text_regions_p.shape[1]) + table_prediction_n = resize_image(table_prediction_n, + text_regions_p.shape[0], + text_regions_p.shape[1]) regions_without_separators_d = (text_regions_p_1_n[:,:] == 1)*1 regions_without_separators_d[table_prediction_n[:,:] == 1] = 1 @@ -3565,7 +3650,8 @@ class Eynollah: pixel_line = 3 img_revised_tab2 = self.add_tables_heuristic_to_layout( - text_regions_p_tables, boxes_d, 0, splitter_y_new_d, peaks_neg_tot_tables_d, text_regions_p_tables, + text_regions_p_tables, boxes_d, 0, splitter_y_new_d, + peaks_neg_tot_tables_d, text_regions_p_tables, num_col_classifier, 0.000005, pixel_line) img_revised_tab2_d,_ = self.check_iou_of_bounding_box_and_contour_for_tables( @@ -3574,8 +3660,9 @@ class Eynollah: img_revised_tab2_d_rotated = np.round(img_revised_tab2_d_rotated) img_revised_tab2_d_rotated = img_revised_tab2_d_rotated.astype(np.int8) - - img_revised_tab2_d_rotated = resize_image(img_revised_tab2_d_rotated, text_regions_p.shape[0], text_regions_p.shape[1]) + img_revised_tab2_d_rotated = resize_image(img_revised_tab2_d_rotated, + text_regions_p.shape[0], + text_regions_p.shape[1]) if np.abs(slope_deskew) < 0.13: img_revised_tab = np.copy(img_revised_tab2[:,:,0]) @@ -3646,7 +3733,8 @@ class Eynollah: ##else: ##regions_fully_np = filter_small_drop_capitals_from_no_patch_layout(regions_fully_np, text_regions_p) - ###regions_fully = boosting_headers_by_longshot_region_segmentation(regions_fully, regions_fully_np, img_only_regions) + ###regions_fully = boosting_headers_by_longshot_region_segmentation(regions_fully, + ### regions_fully_np, img_only_regions) # plt.imshow(regions_fully[:,:,0]) # plt.show() text_regions_p[:, :][regions_fully[:, :, 0] == drop_capital_label_in_full_layout_model] = 4 @@ -3709,7 +3797,10 @@ class Eynollah: min_cont_size_to_be_dilated = 10 if len(contours_only_text_parent)>min_cont_size_to_be_dilated and self.light_version: - cx_conts, cy_conts, x_min_conts, x_max_conts, y_min_conts, y_max_conts, _ = find_new_features_of_contours(contours_only_text_parent) + (cx_conts, cy_conts, + x_min_conts, x_max_conts, + y_min_conts, y_max_conts, + _) = find_new_features_of_contours(contours_only_text_parent) args_cont_located = np.array(range(len(contours_only_text_parent))) diff_y_conts = np.abs(y_max_conts[:]-y_min_conts) @@ -3724,15 +3815,31 @@ class Eynollah: args_cont_located_excluded = args_cont_located[diff_x_ratio>=1.3] args_cont_located_included = args_cont_located[diff_x_ratio<1.3] - contours_only_text_parent_excluded = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]>=1.3]#contours_only_text_parent[diff_x_ratio>=1.3] - contours_only_text_parent_included = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]<1.3]#contours_only_text_parent[diff_x_ratio<1.3] + contours_only_text_parent_excluded = [contours_only_text_parent[ind] + #contours_only_text_parent[diff_x_ratio>=1.3] + for ind in range(len(contours_only_text_parent)) + if diff_x_ratio[ind]>=1.3] + contours_only_text_parent_included = [contours_only_text_parent[ind] + #contours_only_text_parent[diff_x_ratio<1.3] + for ind in range(len(contours_only_text_parent)) + if diff_x_ratio[ind]<1.3] - - cx_conts_excluded = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]>=1.3]#cx_conts[diff_x_ratio>=1.3] - cx_conts_included = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]<1.3]#cx_conts[diff_x_ratio<1.3] - - cy_conts_excluded = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]>=1.3]#cy_conts[diff_x_ratio>=1.3] - cy_conts_included = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]<1.3]#cy_conts[diff_x_ratio<1.3] + cx_conts_excluded = [cx_conts[ind] + #cx_conts[diff_x_ratio>=1.3] + for ind in range(len(cx_conts)) + if diff_x_ratio[ind]>=1.3] + cx_conts_included = [cx_conts[ind] + #cx_conts[diff_x_ratio<1.3] + for ind in range(len(cx_conts)) + if diff_x_ratio[ind]<1.3] + cy_conts_excluded = [cy_conts[ind] + #cy_conts[diff_x_ratio>=1.3] + for ind in range(len(cy_conts)) + if diff_x_ratio[ind]>=1.3] + cy_conts_included = [cy_conts[ind] + #cy_conts[diff_x_ratio<1.3] + for ind in range(len(cy_conts)) + if diff_x_ratio[ind]<1.3] #print(diff_x_ratio, 'ratio') text_regions_p = text_regions_p.astype('uint8') @@ -3754,7 +3861,10 @@ class Eynollah: contours_only_dilated, hir_on_text_dilated = return_contours_of_image(text_regions_p_textregions_dilated) contours_only_dilated = return_parent_contours(contours_only_dilated, hir_on_text_dilated) - indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located = self.return_indexes_of_contours_loctaed_inside_another_list_of_contours(contours_only_dilated, contours_only_text_parent_included, cx_conts_included, cy_conts_included, args_cont_located_included) + indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located = \ + self.return_indexes_of_contours_located_inside_another_list_of_contours( + contours_only_dilated, contours_only_text_parent_included, + cx_conts_included, cy_conts_included, args_cont_located_included) if len(args_cont_located_excluded)>0: @@ -3767,7 +3877,7 @@ class Eynollah: flattened_array = np.concatenate([arr.ravel() for arr in array_list]) #print(len( np.unique(flattened_array)), 'indexes_of_located_cont uniques') - missing_textregions = list( set(np.array(range(len(contours_only_text_parent))) ) - set(np.unique(flattened_array)) ) + missing_textregions = list( set(range(len(contours_only_text_parent))) - set(flattened_array) ) #print(missing_textregions, 'missing_textregions') for ind in missing_textregions: @@ -3887,12 +3997,13 @@ class Eynollah: region_with_curr_order = ordered[ind] if region_with_curr_order < len(contours_only_dilated): if np.isscalar(indexes_of_located_cont[region_with_curr_order]): - org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]] + org_contours_indexes.extend([indexes_of_located_cont[region_with_curr_order]]) else: arg_sort_located_cont = np.argsort(center_y_coordinates_of_located[region_with_curr_order]) - org_contours_indexes = org_contours_indexes + list(np.array(indexes_of_located_cont[region_with_curr_order])[arg_sort_located_cont]) ##org_contours_indexes + list ( + org_contours_indexes.extend( + np.array(indexes_of_located_cont[region_with_curr_order])[arg_sort_located_cont]) else: - org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]] + org_contours_indexes.extend([indexes_of_located_cont[region_with_curr_order]]) region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))] return org_contours_indexes, region_ids @@ -3915,17 +4026,13 @@ class Eynollah: if len(peaks_real)>70: print(len(peaks_real), 'len(peaks_real)') - peaks_real = peaks_real[(peaks_realwidth1)] arg_sort = np.argsort(sum_smoothed[peaks_real]) - arg_sort4 =arg_sort[::-1][:4] - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] argsort_sorted = np.argsort(peaks_sort_4) - first_4_sorted = peaks_sort_4[argsort_sorted] y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] #print(first_4_sorted,'first_4_sorted') @@ -4109,7 +4216,8 @@ class Eynollah: return x_differential_new - def filter_contours_inside_a_bigger_one(self,contours, contours_d_ordered, image, marginal_cnts=None, type_contour="textregion"): + def filter_contours_inside_a_bigger_one(self, contours, contours_d_ordered, image, + marginal_cnts=None, type_contour="textregion"): if type_contour=="textregion": areas = [cv2.contourArea(contours[j]) for j in range(len(contours))] area_tot = image.shape[0]*image.shape[1] @@ -4129,7 +4237,10 @@ class Eynollah: results = [cv2.pointPolygonTest(contours[ind], (cx_main[ind_small], cy_main[ind_small]), False) for ind in contours_index_big] if marginal_cnts: - results_marginal = [cv2.pointPolygonTest(marginal_cnts[ind], (cx_main[ind_small], cy_main[ind_small]), False) + results_marginal = [cv2.pointPolygonTest(marginal_cnts[ind], + (cx_main[ind_small], + cy_main[ind_small]), + False) for ind in range(len(marginal_cnts))] results_marginal = np.array(results_marginal) @@ -4184,7 +4295,10 @@ class Eynollah: args_with_bigger_area = np.array(args_all)[areas_without > 1.5*area_of_con_interest] if len(args_with_bigger_area)>0: - results = [cv2.pointPolygonTest(contours_txtline_of_all_textregions[ind], (cx_main_tot[ij], cy_main_tot[ij]), False) + results = [cv2.pointPolygonTest(contours_txtline_of_all_textregions[ind], + (cx_main_tot[ij], + cy_main_tot[ij]), + False) for ind in args_with_bigger_area ] results = np.array(results) if np.any(results==1): @@ -4196,14 +4310,16 @@ class Eynollah: textregion_index_to_del = np.array(textregion_index_to_del) textline_in_textregion_index_to_del = np.array(textline_in_textregion_index_to_del) for ind_u_a_trs in np.unique(textregion_index_to_del): - textline_in_textregion_index_to_del_ind = textline_in_textregion_index_to_del[textregion_index_to_del==ind_u_a_trs] + textline_in_textregion_index_to_del_ind = \ + textline_in_textregion_index_to_del[textregion_index_to_del==ind_u_a_trs] textline_in_textregion_index_to_del_ind = np.sort(textline_in_textregion_index_to_del_ind)[::-1] for ittrd in textline_in_textregion_index_to_del_ind: contours[ind_u_a_trs].pop(ittrd) return contours - def return_indexes_of_contours_loctaed_inside_another_list_of_contours(self, contours, contours_loc, cx_main_loc, cy_main_loc, indexes_loc): + def return_indexes_of_contours_located_inside_another_list_of_contours( + self, contours, contours_loc, cx_main_loc, cy_main_loc, indexes_loc): indexes_of_located_cont = [] center_x_coordinates_of_located = [] center_y_coordinates_of_located = [] @@ -4217,7 +4333,8 @@ class Eynollah: for ind in range(len(cy_main_loc)) ] results = np.array(results) indexes_in = np.where((results == 0) | (results == 1)) - indexes = indexes_loc[indexes_in]# [(results == 0) | (results == 1)]#np.where((results == 0) | (results == 1)) + # [(results == 0) | (results == 1)]#np.where((results == 0) | (results == 1)) + indexes = indexes_loc[indexes_in] indexes_of_located_cont.append(indexes) center_x_coordinates_of_located.append(np.array(cx_main_loc)[indexes_in] ) @@ -4247,7 +4364,10 @@ class Eynollah: ###contours_with_textline = [] ###for ind_tr, con_tr in enumerate(contours): - ###results = [cv2.pointPolygonTest(con_tr, (cx_main_textline[index_textline_con], cy_main_textline[index_textline_con]), False) + ###results = [cv2.pointPolygonTest(con_tr, + ### (cx_main_textline[index_textline_con], + ### cy_main_textline[index_textline_con]), + ### False) ### for index_textline_con in range(len(contours_txtline_of_all_textregions)) ] ###results = np.array(results) ###if np.any(results==1): @@ -4300,7 +4420,9 @@ class Eynollah: return (slopes_rem, all_found_textline_polygons_rem, boxes_text_rem, txt_con_org_rem, contours_only_text_parent_rem, index_by_text_par_con_rem_sort) - def separate_marginals_to_left_and_right_and_order_from_top_to_down(self, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes_marginals, mid_point_of_page_width): + def separate_marginals_to_left_and_right_and_order_from_top_to_down( + self, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, + slopes_marginals, mid_point_of_page_width): cx_marg, cy_marg, _, _, _, _, _ = find_new_features_of_contours( polygons_of_marginals) @@ -4310,8 +4432,10 @@ class Eynollah: poly_marg_left = list( np.array(polygons_of_marginals)[cx_marg < mid_point_of_page_width] ) poly_marg_right = list( np.array(polygons_of_marginals)[cx_marg >= mid_point_of_page_width] ) - all_found_textline_polygons_marginals_left = list( np.array(all_found_textline_polygons_marginals)[cx_marg < mid_point_of_page_width] ) - all_found_textline_polygons_marginals_right = list( np.array(all_found_textline_polygons_marginals)[cx_marg >= mid_point_of_page_width] ) + all_found_textline_polygons_marginals_left = \ + list( np.array(all_found_textline_polygons_marginals)[cx_marg < mid_point_of_page_width] ) + all_found_textline_polygons_marginals_right = \ + list( np.array(all_found_textline_polygons_marginals)[cx_marg >= mid_point_of_page_width] ) all_box_coord_marginals_left = list( np.array(all_box_coord_marginals)[cx_marg < mid_point_of_page_width] ) all_box_coord_marginals_right = list( np.array(all_box_coord_marginals)[cx_marg >= mid_point_of_page_width] ) @@ -4322,20 +4446,38 @@ class Eynollah: cy_marg_left = cy_marg[cx_marg < mid_point_of_page_width] cy_marg_right = cy_marg[cx_marg >= mid_point_of_page_width] - ordered_left_marginals = [poly for _, poly in sorted(zip(cy_marg_left, poly_marg_left), key=lambda x: x[0])] - ordered_right_marginals = [poly for _, poly in sorted(zip(cy_marg_right, poly_marg_right), key=lambda x: x[0])] + ordered_left_marginals = [poly for _, poly in sorted(zip(cy_marg_left, poly_marg_left), + key=lambda x: x[0])] + ordered_right_marginals = [poly for _, poly in sorted(zip(cy_marg_right, poly_marg_right), + key=lambda x: x[0])] - ordered_left_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_left, all_found_textline_polygons_marginals_left), key=lambda x: x[0])] - ordered_right_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_right, all_found_textline_polygons_marginals_right), key=lambda x: x[0])] + ordered_left_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_left, + all_found_textline_polygons_marginals_left), + key=lambda x: x[0])] + ordered_right_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_right, + all_found_textline_polygons_marginals_right), + key=lambda x: x[0])] - ordered_left_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_left, all_box_coord_marginals_left), key=lambda x: x[0])] - ordered_right_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_right, all_box_coord_marginals_right), key=lambda x: x[0])] + ordered_left_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_left, + all_box_coord_marginals_left), + key=lambda x: x[0])] + ordered_right_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_right, + all_box_coord_marginals_right), + key=lambda x: x[0])] - ordered_left_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_left, slopes_marg_left), key=lambda x: x[0])] - ordered_right_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_right, slopes_marg_right), key=lambda x: x[0])] + ordered_left_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_left, slopes_marg_left), + key=lambda x: x[0])] + ordered_right_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_right, slopes_marg_right), + key=lambda x: x[0])] - return ordered_left_marginals, ordered_right_marginals, ordered_left_marginals_textline, ordered_right_marginals_textline, ordered_left_marginals_bbox, ordered_right_marginals_bbox, ordered_left_slopes_marginals, ordered_right_slopes_marginals - + return (ordered_left_marginals, + ordered_right_marginals, + ordered_left_marginals_textline, + ordered_right_marginals_textline, + ordered_left_marginals_bbox, + ordered_right_marginals_bbox, + ordered_left_slopes_marginals, + ordered_right_slopes_marginals) def run(self, overwrite: bool = False, @@ -4420,9 +4562,11 @@ class Eynollah: self.logger.info(f"Processing file: {self.writer.image_filename}") self.logger.info("Step 1/5: Image Enhancement") - img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) + img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = \ + self.run_enhancement(self.light_version) - self.logger.info(f"Image: {self.image.shape[1]}x{self.image.shape[0]}, {self.dpi} DPI, {num_col_classifier} columns") + self.logger.info(f"Image: {self.image.shape[1]}x{self.image.shape[0]}, " + f"{self.dpi} DPI, {num_col_classifier} columns") if is_image_enhanced: self.logger.info("Enhancement applied") @@ -4433,7 +4577,8 @@ class Eynollah: if self.extract_only_images: self.logger.info("Step 2/5: Image Extraction Mode") - text_regions_p_1, erosion_hurts, polygons_seplines, polygons_of_images, image_page, page_coord, cont_page = \ + text_regions_p_1, erosion_hurts, polygons_seplines, polygons_of_images, \ + image_page, page_coord, cont_page = \ self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], @@ -4465,20 +4610,20 @@ class Eynollah: M_main_tot = [cv2.moments(all_found_textline_polygons[j]) for j in range(len(all_found_textline_polygons))] - w_h_textlines = [cv2.boundingRect(all_found_textline_polygons[j])[2:] for j in range(len(all_found_textline_polygons))] + w_h_textlines = [cv2.boundingRect(all_found_textline_polygons[j])[2:] + for j in range(len(all_found_textline_polygons))] w_h_textlines = [w_h_textlines[j][0] / float(w_h_textlines[j][1]) for j in range(len(w_h_textlines))] cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - all_found_textline_polygons = self.get_textlines_of_a_textregion_sorted(all_found_textline_polygons, cx_main_tot, cy_main_tot, w_h_textlines)#all_found_textline_polygons[::-1] - - all_found_textline_polygons=[ all_found_textline_polygons ] - + all_found_textline_polygons = self.get_textlines_of_a_textregion_sorted( + #all_found_textline_polygons[::-1] + all_found_textline_polygons, cx_main_tot, cy_main_tot, w_h_textlines) + all_found_textline_polygons = [ all_found_textline_polygons ] all_found_textline_polygons = dilate_textline_contours(all_found_textline_polygons) all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( all_found_textline_polygons, None, textline_mask_tot_ea, type_contour="textline") - order_text_new = [0] slopes =[0] id_of_texts_tot =['region_0001'] @@ -4498,15 +4643,23 @@ class Eynollah: if self.ocr and not self.tr: gc.collect() - ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, textline_light=True) + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons, self.prediction_model, + self.b_s_ocr, self.num_to_char, textline_light=True) else: ocr_all_textlines = None pcgts = self.writer.build_pagexml_no_full_layout( cont_page, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals_left, polygons_of_marginals_right, - all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, - cont_page, polygons_seplines, contours_tables, ocr_all_textlines=ocr_all_textlines, conf_contours_textregion=conf_contours_textregions, skip_layout_reading_order=self.skip_layout_and_reading_order) + all_found_textline_polygons, page_coord, polygons_of_images, + polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, contours_tables, + ocr_all_textlines=ocr_all_textlines, + conf_contours_textregion=conf_contours_textregions, + skip_layout_reading_order=self.skip_layout_and_reading_order) self.logger.info("Basic processing complete") return pcgts @@ -4516,7 +4669,8 @@ class Eynollah: if self.light_version: self.logger.info("Using light version processing") - text_regions_p_1 ,erosion_hurts, polygons_seplines, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ + text_regions_p_1 ,erosion_hurts, polygons_seplines, textline_mask_tot_ea, \ + img_bin_light, confidence_matrix = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier) #print("text region early -2 in %.1fs", time.time() - t0) @@ -4528,7 +4682,6 @@ class Eynollah: img_h_new = img_w_new * textline_mask_tot_ea.shape[0] // textline_mask_tot_ea.shape[1] textline_mask_tot_ea_deskew = resize_image(textline_mask_tot_ea,img_h_new, img_w_new ) - slope_deskew = self.run_deskew(textline_mask_tot_ea_deskew) else: slope_deskew = self.run_deskew(textline_mask_tot_ea) @@ -4537,7 +4690,8 @@ class Eynollah: num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ text_regions_p_1, cont_page, table_prediction, textline_mask_tot_ea, img_bin_light = \ self.run_graphics_and_columns_light(text_regions_p_1, textline_mask_tot_ea, - num_col_classifier, num_column_is_classified, erosion_hurts, img_bin_light) + num_col_classifier, num_column_is_classified, + erosion_hurts, img_bin_light) #self.logger.info("run graphics %.1fs ", time.time() - t1t) #print("text region early -3 in %.1fs", time.time() - t0) textline_mask_tot_ea_org = np.copy(textline_mask_tot_ea) @@ -4552,7 +4706,8 @@ class Eynollah: t1 = time.time() num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ text_regions_p_1, cont_page, table_prediction = \ - self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts) + self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, + erosion_hurts) self.logger.info(f"Graphics detection took {time.time() - t1:.1f}s") #self.logger.info('cont_page %s', cont_page) #plt.imshow(table_prediction) @@ -4617,13 +4772,15 @@ class Eynollah: ## birdan sora chock chakir t1 = time.time() if not self.full_layout: - polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, \ + polygons_of_images, img_revised_tab, text_regions_p_1_n, \ + textline_mask_tot_d, regions_without_separators_d, \ boxes, boxes_d, polygons_of_marginals, contours_tables = \ self.run_boxes_no_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, table_prediction, erosion_hurts) ###polygons_of_marginals = dilate_textregion_contours(polygons_of_marginals) else: - polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, \ + polygons_of_images, img_revised_tab, text_regions_p_1_n, \ + textline_mask_tot_d, regions_without_separators_d, \ regions_fully, regions_without_separators, polygons_of_marginals, contours_tables = \ self.run_boxes_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions, table_prediction, erosion_hurts, @@ -4690,8 +4847,10 @@ class Eynollah: areas_cnt_text_d = self.return_list_of_contours_with_desired_order( areas_cnt_text_d, index_con_parents_d) - cx_bigest_d_big, cy_biggest_d_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest_d]) - cx_bigest_d, cy_biggest_d, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent_d) + cx_bigest_d_big, cy_biggest_d_big, _, _, _, _, _ = \ + find_new_features_of_contours([contours_biggest_d]) + cx_bigest_d, cy_biggest_d, _, _, _, _, _ = \ + find_new_features_of_contours(contours_only_text_parent_d) try: if len(cx_bigest_d) >= 5: cx_bigest_d_last5 = cx_bigest_d[-5:] @@ -4751,13 +4910,19 @@ class Eynollah: pcgts = self.writer.build_pagexml_full_layout( [], [], page_coord, [], [], [], [], [], [], polygons_of_images, contours_tables, [], - polygons_of_marginals, polygons_of_marginals, empty_marginals, empty_marginals, empty_marginals, empty_marginals, [], [], [], [], + polygons_of_marginals, polygons_of_marginals, + empty_marginals, empty_marginals, + empty_marginals, empty_marginals, + [], [], [], [], cont_page, polygons_seplines) else: pcgts = self.writer.build_pagexml_no_full_layout( [], page_coord, [], [], [], [], polygons_of_images, - polygons_of_marginals, polygons_of_marginals, empty_marginals, empty_marginals, empty_marginals, empty_marginals, [], [], [], + polygons_of_marginals, polygons_of_marginals, + empty_marginals, empty_marginals, + empty_marginals, empty_marginals, + [], [], [], cont_page, polygons_seplines, contours_tables) return pcgts @@ -4767,7 +4932,8 @@ class Eynollah: if self.light_version: contours_only_text_parent = dilate_textregion_contours(contours_only_text_parent) contours_only_text_parent , contours_only_text_parent_d_ordered = self.filter_contours_inside_a_bigger_one( - contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, marginal_cnts=polygons_of_marginals) + contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, + marginal_cnts=polygons_of_marginals) #print("text region early 3.5 in %.1fs", time.time() - t0) txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) @@ -4793,19 +4959,26 @@ class Eynollah: polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_org, boxes_marginals, slope_deskew) - #slopes, all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con = \ + #slopes, all_found_textline_polygons, boxes_text, txt_con_org, \ + # contours_only_text_parent, index_by_text_par_con = \ # self.delete_regions_without_textlines(slopes, all_found_textline_polygons, # boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con) - #slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, polygons_of_marginals, polygons_of_marginals, _ = \ + #slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, \ + # polygons_of_marginals, polygons_of_marginals, _ = \ # self.delete_regions_without_textlines(slopes_marginals, all_found_textline_polygons_marginals, - # boxes_marginals, polygons_of_marginals, polygons_of_marginals, np.array(range(len(polygons_of_marginals)))) - all_found_textline_polygons = dilate_textline_contours(all_found_textline_polygons) + # boxes_marginals, polygons_of_marginals, polygons_of_marginals, + # np.array(range(len(polygons_of_marginals)))) + all_found_textline_polygons = dilate_textline_contours( + all_found_textline_polygons) all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( all_found_textline_polygons, None, textline_mask_tot_ea_org, type_contour="textline") - all_found_textline_polygons_marginals = dilate_textline_contours(all_found_textline_polygons_marginals) - contours_only_text_parent, txt_con_org, conf_contours_textregions, all_found_textline_polygons, contours_only_text_parent_d_ordered, \ + all_found_textline_polygons_marginals = dilate_textline_contours( + all_found_textline_polygons_marginals) + contours_only_text_parent, txt_con_org, conf_contours_textregions, \ + all_found_textline_polygons, contours_only_text_parent_d_ordered, \ index_by_text_par_con = self.filter_contours_without_textline_inside( - contours_only_text_parent, txt_con_org, all_found_textline_polygons, contours_only_text_parent_d_ordered, conf_contours_textregions) + contours_only_text_parent, txt_con_org, all_found_textline_polygons, + contours_only_text_parent_d_ordered, conf_contours_textregions) else: textline_mask_tot_ea = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1) all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, \ @@ -4847,7 +5020,13 @@ class Eynollah: all_found_textline_polygons_marginals, textline_mask_tot_ea, num_col_classifier) mid_point_of_page_width = text_regions_p.shape[1] / 2. - polygons_of_marginals_left, polygons_of_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes_marginals_left, slopes_marginals_right = self.separate_marginals_to_left_and_right_and_order_from_top_to_down(polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes_marginals, mid_point_of_page_width) + (polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes_marginals_left, slopes_marginals_right) = \ + self.separate_marginals_to_left_and_right_and_order_from_top_to_down( + polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, + slopes_marginals, mid_point_of_page_width) #print(len(polygons_of_marginals), len(ordered_left_marginals), len(ordered_right_marginals), 'marginals ordred') if self.full_layout: @@ -4871,40 +5050,41 @@ class Eynollah: all_found_textline_polygons, all_found_textline_polygons_h, slopes, slopes_h, \ contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, \ conf_contours_textregions, conf_contours_textregions_h = fun( - text_regions_p, regions_fully, contours_only_text_parent, - all_box_coord, all_found_textline_polygons, slopes, contours_only_text_parent_d_ordered, conf_contours_textregions) + text_regions_p, regions_fully, contours_only_text_parent, + all_box_coord, all_found_textline_polygons, + slopes, contours_only_text_parent_d_ordered, conf_contours_textregions) if self.plotter: self.plotter.save_plot_of_layout(text_regions_p, image_page) self.plotter.save_plot_of_layout_all(text_regions_p, image_page) - pixel_img = 4 - polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img) + label_img = 4 + polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, label_img) ##all_found_textline_polygons = adhere_drop_capital_region_into_corresponding_textline( ##text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, ##all_box_coord, all_box_coord_h, all_found_textline_polygons, all_found_textline_polygons_h, ##kernel=KERNEL, curved_line=self.curved_line, textline_light=self.textline_light) if not self.reading_order_machine_based: - pixel_seps = 6 + label_seps = 6 if not self.headers_off: if np.abs(slope_deskew) < SLOPE_THRESHOLD: num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps, contours_only_text_parent_h) + num_col_classifier, self.tables, label_seps, contours_only_text_parent_h) else: _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps, contours_only_text_parent_h_d_ordered) + num_col_classifier, self.tables, label_seps, contours_only_text_parent_h_d_ordered) elif self.headers_off: if np.abs(slope_deskew) < SLOPE_THRESHOLD: num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps) + num_col_classifier, self.tables, label_seps) else: _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps) + num_col_classifier, self.tables, label_seps) if num_col_classifier >= 3: if np.abs(slope_deskew) < SLOPE_THRESHOLD: @@ -4949,7 +5129,8 @@ class Eynollah: contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) else: order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) + contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, + boxes_d, textline_mask_tot_d) self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") if self.ocr and not self.tr: @@ -4962,27 +5143,37 @@ class Eynollah: gc.collect() if len(all_found_textline_polygons)>0: - ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines = None if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: - ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_left, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons_marginals_left, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_marginals_left = None if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: - ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_right, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons_marginals_right, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_marginals_right = None if all_found_textline_polygons_h and len(all_found_textline_polygons)>0: - ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_h, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons_h, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_h = None if polygons_of_drop_capitals and len(polygons_of_drop_capitals)>0: - ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines(image_page, polygons_of_drop_capitals, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines( + image_page, polygons_of_drop_capitals, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_drop = None else: @@ -4997,9 +5188,15 @@ class Eynollah: pcgts = self.writer.build_pagexml_full_layout( contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, - polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals_left, polygons_of_marginals_right, - all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, - cont_page, polygons_seplines, ocr_all_textlines, ocr_all_textlines_h, ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, ocr_all_textlines_drop, conf_contours_textregions, conf_contours_textregions_h) + polygons_of_images, contours_tables, polygons_of_drop_capitals, + polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, ocr_all_textlines, ocr_all_textlines_h, + ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, + ocr_all_textlines_drop, + conf_contours_textregions, conf_contours_textregions_h) return pcgts @@ -5034,18 +5231,14 @@ class Eynollah: if self.ocr and self.tr: self.logger.info("Step 4.5/5: OCR Processing") - if torch.cuda.is_available(): self.logger.info("Using GPU acceleration") else: self.logger.info("Using CPU processing") - if self.light_version: self.logger.info("Using light version OCR") - if self.textline_light: self.logger.info("Using light text line detection for OCR") - self.logger.info("Processing text lines...") device = cuda.get_current_device() @@ -5090,7 +5283,8 @@ class Eynollah: img_croped = img_poly_on_img[y:y+h, x:x+w, :] #cv2.imwrite('./extracted_lines/'+str(ind_tot)+'.jpg', img_croped) - text_ocr = self.return_ocr_of_textline_without_common_section(img_croped, model_ocr, processor, device, w, h2w_ratio, ind_tot) + text_ocr = self.return_ocr_of_textline_without_common_section( + img_croped, model_ocr, processor, device, w, h2w_ratio, ind_tot) ocr_textline_in_textregion.append(text_ocr) ind_tot = ind_tot +1 ocr_all_textlines.append(ocr_textline_in_textregion) @@ -5098,13 +5292,19 @@ class Eynollah: elif self.ocr and not self.tr: gc.collect() if len(all_found_textline_polygons)>0: - ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: - ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_left, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons_marginals_left, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: - ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines(image_page, all_found_textline_polygons_marginals_right, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines( + image_page, all_found_textline_polygons_marginals_right, self.prediction_model, + self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines = None @@ -5117,9 +5317,14 @@ class Eynollah: pcgts = self.writer.build_pagexml_no_full_layout( txt_con_org, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals_left, polygons_of_marginals_right, - all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, - cont_page, polygons_seplines, contours_tables, ocr_all_textlines, ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, conf_contours_textregions) + all_found_textline_polygons, all_box_coord, polygons_of_images, + polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, contours_tables, ocr_all_textlines, + ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, + conf_contours_textregions) return pcgts @@ -5138,7 +5343,6 @@ class Eynollah_ocr: min_conf_value_of_textline_text : Optional[float]=None, logger=None, ): - self.dir_models = dir_models self.model_name = model_name self.tr_ocr = tr_ocr self.export_textline_images_and_text = export_textline_images_and_text @@ -5261,7 +5465,9 @@ class Eynollah_ocr: if child_textlines.tag.endswith("Coords"): cropped_lines_region_indexer.append(indexer_text_region) p_h=child_textlines.attrib['points'].split(' ') - textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) + textline_coords = np.array( [ [int(x.split(',')[0]), + int(x.split(',')[1]) ] + for x in p_h] ) x,y,w,h = cv2.boundingRect(textline_coords) if dir_out_image_text: @@ -5277,9 +5483,12 @@ class Eynollah_ocr: img_crop = img_poly_on_img[y:y+h, x:x+w, :] img_crop[mask_poly==0] = 255 - self.logger.debug("processing %d lines for '%s'", len(cropped_lines), nn.attrib['id']) + self.logger.debug("processing %d lines for '%s'", + len(cropped_lines), nn.attrib['id']) if h2w_ratio > 0.1: - cropped_lines.append(resize_image(img_crop, tr_ocr_input_height_and_width, tr_ocr_input_height_and_width) ) + cropped_lines.append(resize_image(img_crop, + tr_ocr_input_height_and_width, + tr_ocr_input_height_and_width) ) cropped_lines_meging_indexing.append(0) indexer_b_s+=1 if indexer_b_s==self.b_s: @@ -5288,8 +5497,10 @@ class Eynollah_ocr: indexer_b_s = 0 pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values - generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) - generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + generated_ids_merged = self.model_ocr.generate( + pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode( + generated_ids_merged, skip_special_tokens=True) extracted_texts = extracted_texts + generated_text_merged @@ -5297,7 +5508,9 @@ class Eynollah_ocr: splited_images, _ = return_textlines_split_if_needed(img_crop, None) #print(splited_images) if splited_images: - cropped_lines.append(resize_image(splited_images[0], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) + cropped_lines.append(resize_image(splited_images[0], + tr_ocr_input_height_and_width, + tr_ocr_input_height_and_width)) cropped_lines_meging_indexing.append(1) indexer_b_s+=1 @@ -5307,13 +5520,17 @@ class Eynollah_ocr: indexer_b_s = 0 pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values - generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) - generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + generated_ids_merged = self.model_ocr.generate( + pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode( + generated_ids_merged, skip_special_tokens=True) extracted_texts = extracted_texts + generated_text_merged - cropped_lines.append(resize_image(splited_images[1], tr_ocr_input_height_and_width, tr_ocr_input_height_and_width)) + cropped_lines.append(resize_image(splited_images[1], + tr_ocr_input_height_and_width, + tr_ocr_input_height_and_width)) cropped_lines_meging_indexing.append(-1) indexer_b_s+=1 @@ -5323,8 +5540,10 @@ class Eynollah_ocr: indexer_b_s = 0 pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values - generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) - generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + generated_ids_merged = self.model_ocr.generate( + pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode( + generated_ids_merged, skip_special_tokens=True) extracted_texts = extracted_texts + generated_text_merged @@ -5339,8 +5558,10 @@ class Eynollah_ocr: indexer_b_s = 0 pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values - generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) - generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + generated_ids_merged = self.model_ocr.generate( + pixel_values_merged.to(self.device)) + generated_text_merged = self.processor.batch_decode( + generated_ids_merged, skip_special_tokens=True) extracted_texts = extracted_texts + generated_text_merged @@ -5371,15 +5592,22 @@ class Eynollah_ocr: ####n_end = (i+1)*self.b_s ####imgs = cropped_lines[n_start:n_end] ####pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values - ####generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) - ####generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) + ####generated_ids_merged = self.model_ocr.generate( + #### pixel_values_merged.to(self.device)) + ####generated_text_merged = self.processor.batch_decode( + #### generated_ids_merged, skip_special_tokens=True) ####extracted_texts = extracted_texts + generated_text_merged del cropped_lines gc.collect() - extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + extracted_texts_merged = [extracted_texts[ind] + if cropped_lines_meging_indexing[ind]==0 + else extracted_texts[ind]+" "+extracted_texts[ind+1] + if cropped_lines_meging_indexing[ind]==1 + else None + for ind in range(len(cropped_lines_meging_indexing))] extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] #print(extracted_texts_merged, len(extracted_texts_merged)) @@ -5401,7 +5629,8 @@ class Eynollah_ocr: w_bb = bb_ind[2] h_bb = bb_ind[3] - font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font.path, w_bb, int(h_bb*0.4) ) + font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], + font.path, w_bb, int(h_bb*0.4) ) ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) @@ -5419,25 +5648,27 @@ class Eynollah_ocr: #print(len(unique_cropped_lines_region_indexer), 'unique_cropped_lines_region_indexer') #######text_by_textregion = [] #######for ind in unique_cropped_lines_region_indexer: - #######extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] - + #######ind = np.array(cropped_lines_region_indexer)==ind + #######extracted_texts_merged_un = np.array(extracted_texts_merged)[ind] #######text_by_textregion.append(" ".join(extracted_texts_merged_un)) text_by_textregion = [] for ind in unique_cropped_lines_region_indexer: - extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] + ind = np.array(cropped_lines_region_indexer) == ind + extracted_texts_merged_un = np.array(extracted_texts_merged)[ind] if len(extracted_texts_merged_un)>1: text_by_textregion_ind = "" next_glue = "" for indt in range(len(extracted_texts_merged_un)): - if extracted_texts_merged_un[indt].endswith('⸗') or extracted_texts_merged_un[indt].endswith('-') or extracted_texts_merged_un[indt].endswith('¬'): - text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt][:-1] + if (extracted_texts_merged_un[indt].endswith('⸗') or + extracted_texts_merged_un[indt].endswith('-') or + extracted_texts_merged_un[indt].endswith('¬')): + text_by_textregion_ind += next_glue + extracted_texts_merged_un[indt][:-1] next_glue = "" else: - text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt] + text_by_textregion_ind += next_glue + extracted_texts_merged_un[indt] next_glue = " " text_by_textregion.append(text_by_textregion_ind) - else: text_by_textregion.append(" ".join(extracted_texts_merged_un)) @@ -5495,7 +5726,9 @@ class Eynollah_ocr: unicode_textregion.text = text_by_textregion[indexer_textregion] indexer_textregion = indexer_textregion + 1 - ###sample_order = [(id_to_order[tid], text) for tid, text in zip(id_textregions, textregions_by_existing_ids) if tid in id_to_order] + ###sample_order = [(id_to_order[tid], text) + ### for tid, text in zip(id_textregions, textregions_by_existing_ids) + ### if tid in id_to_order] ##ordered_texts_sample = [text for _, text in sorted(sample_order)] ##tot_page_text = ' '.join(ordered_texts_sample) @@ -5569,7 +5802,9 @@ class Eynollah_ocr: if child_textlines.tag.endswith("Coords"): cropped_lines_region_indexer.append(indexer_text_region) p_h=child_textlines.attrib['points'].split(' ') - textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) + textline_coords = np.array( [ [int(x.split(',')[0]), + int(x.split(',')[1]) ] + for x in p_h] ) x,y,w,h = cv2.boundingRect(textline_coords) @@ -5601,17 +5836,19 @@ class Eynollah_ocr: img_crop[mask_poly==0] = 255 else: - #print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi') + # print(file_name, angle_degrees, w*h, + # mask_poly[:,:,0].sum(), + # mask_poly[:,:,0].sum() /float(w*h) , + # 'didi') if angle_degrees > 3: better_des_slope = get_orientation_moments(textline_coords) - img_crop = rotate_image_with_padding(img_crop, better_des_slope ) - + img_crop = rotate_image_with_padding(img_crop, better_des_slope) if dir_in_bin is not None: - img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope ) + img_crop_bin = rotate_image_with_padding(img_crop_bin, better_des_slope) - mask_poly = rotate_image_with_padding(mask_poly, better_des_slope ) + mask_poly = rotate_image_with_padding(mask_poly, better_des_slope) mask_poly = mask_poly.astype('uint8') #new bounding box @@ -5622,7 +5859,6 @@ class Eynollah_ocr: if not self.do_not_mask_with_textline_contour: img_crop[mask_poly==0] = 255 - if dir_in_bin is not None: img_crop_bin = img_crop_bin[y_n:y_n+h_n, x_n:x_n+w_n, :] if not self.do_not_mask_with_textline_contour: @@ -5630,11 +5866,14 @@ class Eynollah_ocr: if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.50 and w_scaled > 90: if dir_in_bin is not None: - img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) + img_crop, img_crop_bin = \ + break_curved_line_into_small_pieces_and_then_merge( + img_crop, mask_poly, img_crop_bin) else: - img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + img_crop, _ = \ + break_curved_line_into_small_pieces_and_then_merge( + img_crop, mask_poly) - else: better_des_slope = 0 if not self.do_not_mask_with_textline_contour: @@ -5647,13 +5886,18 @@ class Eynollah_ocr: else: if mask_poly[:,:,0].sum() /float(w*h) < 0.50 and w_scaled > 90: if dir_in_bin is not None: - img_crop, img_crop_bin = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly, img_crop_bin) + img_crop, img_crop_bin = \ + break_curved_line_into_small_pieces_and_then_merge( + img_crop, mask_poly, img_crop_bin) else: - img_crop, _ = break_curved_line_into_small_pieces_and_then_merge(img_crop, mask_poly) + img_crop, _ = \ + break_curved_line_into_small_pieces_and_then_merge( + img_crop, mask_poly) if not self.export_textline_images_and_text: if w_scaled < 750:#1.5*image_width: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + img_crop, image_height, image_width) cropped_lines.append(img_fin) if abs(better_des_slope) > 45: cropped_lines_ver_index.append(1) @@ -5662,13 +5906,15 @@ class Eynollah_ocr: cropped_lines_meging_indexing.append(0) if dir_in_bin is not None: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) else: splited_images, splited_images_bin = return_textlines_split_if_needed( img_crop, img_crop_bin if dir_in_bin is not None else None) if splited_images: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + splited_images[0], image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(1) @@ -5677,7 +5923,8 @@ class Eynollah_ocr: else: cropped_lines_ver_index.append(0) - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + splited_images[1], image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(-1) @@ -5688,13 +5935,16 @@ class Eynollah_ocr: cropped_lines_ver_index.append(0) if dir_in_bin is not None: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[0], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + splited_images_bin[0], image_height, image_width) cropped_lines_bin.append(img_fin) - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[1], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + splited_images_bin[1], image_height, image_width) cropped_lines_bin.append(img_fin) else: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + img_crop, image_height, image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) @@ -5704,7 +5954,8 @@ class Eynollah_ocr: cropped_lines_ver_index.append(0) if dir_in_bin is not None: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model( + img_crop_bin, image_height, image_width) cropped_lines_bin.append(img_fin) if self.export_textline_images_and_text: @@ -5716,7 +5967,8 @@ class Eynollah_ocr: if cheild_text.tag.endswith("Unicode"): textline_text = cheild_text.text if textline_text: - base_name = os.path.join(dir_out, file_name + '_line_' + str(indexer_textlines)) + base_name = os.path.join( + dir_out, file_name + '_line_' + str(indexer_textlines)) if self.pref_of_dataset: base_name += '_' + self.pref_of_dataset if not self.do_not_mask_with_textline_contour: @@ -5806,25 +6058,31 @@ class Eynollah_ocr: preds_max_fliped = np.max(preds_flipped, axis=2 ) preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=self.end_character - masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) + masked_means_flipped = \ + np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / \ + np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) masked_means_flipped[np.isnan(masked_means_flipped)] = 0 preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) pred_max_not_unk_mask_bool = preds_max_args[:,:]!=self.end_character - masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) + masked_means = \ + np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / \ + np.sum(pred_max_not_unk_mask_bool, axis=1) masked_means[np.isnan(masked_means)] = 0 masked_means_ver = masked_means[indices_ver] #print(masked_means_ver, 'pred_max_not_unk') - indices_where_flipped_conf_value_is_higher = np.where(masked_means_flipped > masked_means_ver)[0] + indices_where_flipped_conf_value_is_higher = \ + np.where(masked_means_flipped > masked_means_ver)[0] #print(indices_where_flipped_conf_value_is_higher, 'indices_where_flipped_conf_value_is_higher') if len(indices_where_flipped_conf_value_is_higher)>0: indices_to_be_replaced = indices_ver[indices_where_flipped_conf_value_is_higher] - preds[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] + preds[indices_to_be_replaced,:,:] = \ + preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] if dir_in_bin is not None: preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) @@ -5833,35 +6091,42 @@ class Eynollah_ocr: preds_max_fliped = np.max(preds_flipped, axis=2 ) preds_max_args_flipped = np.argmax(preds_flipped, axis=2 ) pred_max_not_unk_mask_bool_flipped = preds_max_args_flipped[:,:]!=self.end_character - masked_means_flipped = np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) + masked_means_flipped = \ + np.sum(preds_max_fliped * pred_max_not_unk_mask_bool_flipped, axis=1) / \ + np.sum(pred_max_not_unk_mask_bool_flipped, axis=1) masked_means_flipped[np.isnan(masked_means_flipped)] = 0 preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) pred_max_not_unk_mask_bool = preds_max_args[:,:]!=self.end_character - masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) + masked_means = \ + np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / \ + np.sum(pred_max_not_unk_mask_bool, axis=1) masked_means[np.isnan(masked_means)] = 0 masked_means_ver = masked_means[indices_ver] #print(masked_means_ver, 'pred_max_not_unk') - indices_where_flipped_conf_value_is_higher = np.where(masked_means_flipped > masked_means_ver)[0] + indices_where_flipped_conf_value_is_higher = \ + np.where(masked_means_flipped > masked_means_ver)[0] #print(indices_where_flipped_conf_value_is_higher, 'indices_where_flipped_conf_value_is_higher') if len(indices_where_flipped_conf_value_is_higher)>0: indices_to_be_replaced = indices_ver[indices_where_flipped_conf_value_is_higher] - preds_bin[indices_to_be_replaced,:,:] = preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] + preds_bin[indices_to_be_replaced,:,:] = \ + preds_flipped[indices_where_flipped_conf_value_is_higher, :, :] preds = (preds + preds_bin) / 2. - pred_texts = decode_batch_predictions(preds, self.num_to_char) preds_max = np.max(preds, axis=2 ) preds_max_args = np.argmax(preds, axis=2 ) pred_max_not_unk_mask_bool = preds_max_args[:,:]!=self.end_character - masked_means = np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / np.sum(pred_max_not_unk_mask_bool, axis=1) + masked_means = \ + np.sum(preds_max * pred_max_not_unk_mask_bool, axis=1) / \ + np.sum(pred_max_not_unk_mask_bool, axis=1) for ib in range(imgs.shape[0]): pred_texts_ib = pred_texts[ib].replace("[UNK]", "") @@ -5876,31 +6141,40 @@ class Eynollah_ocr: del cropped_lines_bin gc.collect() - extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + extracted_texts_merged = [extracted_texts[ind] + if cropped_lines_meging_indexing[ind]==0 + else extracted_texts[ind]+" "+extracted_texts[ind+1] + if cropped_lines_meging_indexing[ind]==1 + else None + for ind in range(len(cropped_lines_meging_indexing))] - extracted_conf_value_merged = [extracted_conf_value[ind] if cropped_lines_meging_indexing[ind]==0 else (extracted_conf_value[ind]+extracted_conf_value[ind+1])/2. if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + extracted_conf_value_merged = [extracted_conf_value[ind] + if cropped_lines_meging_indexing[ind]==0 + else (extracted_conf_value[ind]+extracted_conf_value[ind+1])/2. + if cropped_lines_meging_indexing[ind]==1 + else None + for ind in range(len(cropped_lines_meging_indexing))] - extracted_conf_value_merged = [extracted_conf_value_merged[ind_cfm] for ind_cfm in range(len(extracted_texts_merged)) if extracted_texts_merged[ind_cfm] is not None] + extracted_conf_value_merged = [extracted_conf_value_merged[ind_cfm] + for ind_cfm in range(len(extracted_texts_merged)) + if extracted_texts_merged[ind_cfm] is not None] extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - if dir_out_image_text: - #font_path = "Charis-7.000/Charis-Regular.ttf" # Make sure this file exists! font = importlib_resources.files(__package__) / "Charis-Regular.ttf" with importlib_resources.as_file(font) as font: font = ImageFont.truetype(font=font, size=40) for indexer_text, bb_ind in enumerate(total_bb_coordinates): - - x_bb = bb_ind[0] y_bb = bb_ind[1] w_bb = bb_ind[2] h_bb = bb_ind[3] - font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], font.path, w_bb, int(h_bb*0.4) ) + font = fit_text_single_line(draw, extracted_texts_merged[indexer_text], + font.path, w_bb, int(h_bb*0.4) ) ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) @@ -5917,24 +6191,25 @@ class Eynollah_ocr: text_by_textregion = [] for ind in unique_cropped_lines_region_indexer: - extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] + ind = np.array(cropped_lines_region_indexer)==ind + extracted_texts_merged_un = np.array(extracted_texts_merged)[ind] if len(extracted_texts_merged_un)>1: text_by_textregion_ind = "" next_glue = "" for indt in range(len(extracted_texts_merged_un)): - if extracted_texts_merged_un[indt].endswith('⸗') or extracted_texts_merged_un[indt].endswith('-') or extracted_texts_merged_un[indt].endswith('¬'): - text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt][:-1] + if (extracted_texts_merged_un[indt].endswith('⸗') or + extracted_texts_merged_un[indt].endswith('-') or + extracted_texts_merged_un[indt].endswith('¬')): + text_by_textregion_ind += next_glue + extracted_texts_merged_un[indt][:-1] next_glue = "" else: - text_by_textregion_ind = text_by_textregion_ind + next_glue + extracted_texts_merged_un[indt] + text_by_textregion_ind += next_glue + extracted_texts_merged_un[indt] next_glue = " " text_by_textregion.append(text_by_textregion_ind) - else: text_by_textregion.append(" ".join(extracted_texts_merged_un)) #print(text_by_textregion, 'text_by_textregiontext_by_textregiontext_by_textregiontext_by_textregiontext_by_textregion') - - + ###index_tot_regions = [] ###tot_region_ref = [] @@ -5983,7 +6258,8 @@ class Eynollah_ocr: if childtest3.tag.endswith("TextEquiv"): for child_uc in childtest3: if child_uc.tag.endswith("Unicode"): - childtest3.set('conf', f"{extracted_conf_value_merged[indexer]:.2f}") + childtest3.set('conf', + f"{extracted_conf_value_merged[indexer]:.2f}") child_uc.text = extracted_texts_merged[indexer] indexer = indexer + 1 @@ -5999,7 +6275,9 @@ class Eynollah_ocr: unicode_textregion.text = text_by_textregion[indexer_textregion] indexer_textregion = indexer_textregion + 1 - ###sample_order = [(id_to_order[tid], text) for tid, text in zip(id_textregions, textregions_by_existing_ids) if tid in id_to_order] + ###sample_order = [(id_to_order[tid], text) + ### for tid, text in zip(id_textregions, textregions_by_existing_ids) + ### if tid in id_to_order] ##ordered_texts_sample = [text for _, text in sorted(sample_order)] ##tot_page_text = ' '.join(ordered_texts_sample) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index f8926cf..52bf3ef 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1012,8 +1012,13 @@ def check_any_text_region_in_model_one_is_main_or_header_light( (regions_model_full[:,:,0]==2)).sum() pixels_main = all_pixels - pixels_header - if ( (pixels_header/float(pixels_main)>=0.6) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ) and ( (length_con[ii]/float(height_con[ii]) )<=3 )) or ( (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=3 ) ): - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 + if (( pixels_header / float(pixels_main) >= 0.6 and + length_con[ii] / float(height_con[ii]) >= 1.3 and + length_con[ii] / float(height_con[ii]) <= 3 ) or + ( pixels_header / float(pixels_main) >= 0.3 and + length_con[ii] / float(height_con[ii]) >=3 )): + + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ] = 2 contours_only_text_parent_head.append(contours_only_text_parent[ii]) conf_contours_head.append(None) # why not conf_contours[ii], too? if contours_only_text_parent_d_ordered is not None: @@ -1021,8 +1026,9 @@ def check_any_text_region_in_model_one_is_main_or_header_light( all_box_coord_head.append(all_box_coord[ii]) slopes_head.append(slopes[ii]) all_found_textline_polygons_head.append(all_found_textline_polygons[ii]) + else: - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ] = 1 contours_only_text_parent_main.append(contours_only_text_parent[ii]) conf_contours_main.append(conf_contours[ii]) if contours_only_text_parent_d_ordered is not None: @@ -1883,7 +1889,8 @@ def return_boxes_of_images_by_order_of_reading_new( range(x_start_without_mother[dj], x_end_without_mother[dj])) columns_not_covered = list(all_columns - columns_covered_by_mothers) - y_type_2 = np.append(y_type_2, np.ones(len(columns_not_covered) + len(x_start_without_mother), + y_type_2 = np.append(y_type_2, np.ones(len(columns_not_covered) + + len(x_start_without_mother), dtype=int) * splitter_y_new[i]) ##y_lines_by_order = np.append(y_lines_by_order, [splitter_y_new[i]] * len(columns_not_covered)) ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) @@ -1938,7 +1945,8 @@ def return_boxes_of_images_by_order_of_reading_new( columns_covered_by_with_child_no_mothers.update( range(x_start_with_child_without_mother[dj], x_end_with_child_without_mother[dj])) - columns_not_covered_child_no_mother = list(all_columns - columns_covered_by_with_child_no_mothers) + columns_not_covered_child_no_mother = list( + all_columns - columns_covered_by_with_child_no_mothers) #indexes_to_be_spanned=[] for i_s in range(len(x_end_with_child_without_mother)): columns_not_covered_child_no_mother.append(x_start_with_child_without_mother[i_s]) @@ -1948,7 +1956,8 @@ def return_boxes_of_images_by_order_of_reading_new( x_start_with_child_without_mother = np.array(x_start_with_child_without_mother, int) for i_s_nc in columns_not_covered_child_no_mother: if i_s_nc in x_start_with_child_without_mother: - x_end_biggest_column = x_end_with_child_without_mother[x_start_with_child_without_mother==i_s_nc][0] + x_end_biggest_column = \ + x_end_with_child_without_mother[x_start_with_child_without_mother==i_s_nc][0] args_all_biggest_lines = ind_args[(x_starting==i_s_nc) & (x_ending==x_end_biggest_column)] y_column_nc = y_type_2[args_all_biggest_lines] @@ -1996,9 +2005,12 @@ def return_boxes_of_images_by_order_of_reading_new( np.array(list(set(list(range(len(y_all_between_nm_wc)))) - set(list(index_lines_so_close_to_top_separator)))) if len(indexes_remained_after_deleting_closed_lines) > 0: - y_all_between_nm_wc = y_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] - x_starting_all_between_nm_wc = x_starting_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] - x_ending_all_between_nm_wc = x_ending_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + y_all_between_nm_wc = \ + y_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + x_starting_all_between_nm_wc = \ + x_starting_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + x_ending_all_between_nm_wc = \ + x_ending_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] y_all_between_nm_wc = np.append(y_all_between_nm_wc, y_column_nc[i_c]) x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, i_s_nc) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 7a8926d..d41dda1 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -67,7 +67,8 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) neg_peaks_max = np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] + arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[ + y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -78,11 +79,14 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): clusters_to_be_deleted = [] if len(arg_diff_cluster) > 0: - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) + clusters_to_be_deleted.append( + arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : - arg_diff_cluster[i + 1] + 1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + clusters_to_be_deleted.append( + arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : + arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append( + arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] for m in range(len(clusters_to_be_deleted)): @@ -179,7 +183,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - arg_neg_must_be_deleted= np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3] + arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[ + y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3] diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -239,7 +244,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): try: neg_peaks_max=np.max(y_padded_smoothed[peaks]) - arg_neg_must_be_deleted= np.arange(len(peaks_neg))[y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.42] + arg_neg_must_be_deleted = np.arange(len(peaks_neg))[ + y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.42] diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -316,23 +322,36 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): if peaks_values[jj]>mean_value_of_peaks-std_value_of_peaks/2.: point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down =y_max_cont-1##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_down =y_max_cont-1 + ##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) + #point_up + # np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) else: point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down =y_max_cont-1##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_down =y_max_cont-1 + ##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) + #point_up + # np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) point_down_narrow = peaks[jj] + first_nonzero + int( - 1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + 1.4 * dis_to_next_down) + ###-int(dis_to_next_down*1./2) else: dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) if peaks_values[jj]>mean_value_of_peaks-std_value_of_peaks/2.: - point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) + ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) else: - point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) + ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) point_down_narrow = peaks[jj] + first_nonzero + int( 1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) @@ -341,7 +360,9 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_down_narrow = img_patch.shape[0] - 2 - distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + distances = [cv2.pointPolygonTest(contour_text_interest_copy, + tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), + True) for mj in range(len(xv))] distances = np.array(distances) @@ -468,7 +489,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_up =peaks[jj] + first_nonzero - int(1. / 1.8 * dis_to_next) distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), + True) for mj in range(len(xv))] distances = np.array(distances) @@ -543,7 +565,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_down = peaks[jj] + first_nonzero + int(1. / 1.9 * dis_to_next_down) distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), + True) for mj in range(len(xv))] distances = np.array(distances) @@ -613,7 +636,8 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): neg_peaks_max = np.max(y_padded_up_to_down_padded[peaks_neg]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.42] + arg_neg_must_be_deleted = np.arange(len(peaks_neg))[ + y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.42] diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -689,30 +713,50 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) if peaks_values[jj] > mean_value_of_peaks - std_value_of_peaks / 2.0: - point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = x_max_cont - 1 ##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) + ##+int(dis_to_next_up*1./4.0) + point_down = x_max_cont - 1 + ##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) + #point_up + # np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) else: - point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = x_max_cont - 1 ##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) + ##+int(dis_to_next_up*1./4.0) + point_down = x_max_cont - 1 + ##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) + #point_up + # np.max(y_cont) + #peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) - point_down_narrow = peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + point_down_narrow = peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) + ###-int(dis_to_next_down*1./2) else: dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) if peaks_values[jj] > mean_value_of_peaks - std_value_of_peaks / 2.0: - point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) + ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) else: - point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) + ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) + ###-int(dis_to_next_down*1./4.0) - point_down_narrow = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + point_down_narrow = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) + ###-int(dis_to_next_down*1./2) if point_down_narrow >= img_patch.shape[0]: point_down_narrow = img_patch.shape[0] - 2 - distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) for mj in range(len(xv))] + distances = [cv2.pointPolygonTest(contour_text_interest_copy, + tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), + True) + for mj in range(len(xv))] distances = np.array(distances) xvinside = xv[distances >= 0] @@ -801,7 +845,8 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): point_up = peaks[jj] + first_nonzero - int(1.0 / 1.8 * dis_to_next) distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), + True) for mj in range(len(xv))] distances = np.array(distances) @@ -866,7 +911,8 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): point_down = peaks[jj] + first_nonzero + int(1.0 / 1.9 * dis_to_next_down) distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), + True) for mj in range(len(xv))] distances = np.array(distances) @@ -950,7 +996,8 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) neg_peaks_max = np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] + arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[ + y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -963,8 +1010,11 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): if len(arg_diff_cluster) > 0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + clusters_to_be_deleted.append( + arg_neg_must_be_deleted[arg_diff_cluster[i] + 1: + arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append( + arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] for m in range(len(clusters_to_be_deleted)): @@ -1014,7 +1064,8 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): try: neg_peaks_max = np.max(y_padded_smoothed[peaks]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.24] + arg_neg_must_be_deleted = np.arange(len(peaks_neg))[ + y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.24] diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -1290,7 +1341,9 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i return None, cont_final -def textline_contours_postprocessing(textline_mask, slope, contour_text_interest, box_ind, add_boxes_coor_into_textlines=False): +def textline_contours_postprocessing(textline_mask, slope, + contour_text_interest, box_ind, + add_boxes_coor_into_textlines=False): textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 textline_mask = textline_mask.astype(np.uint8) kernel = np.ones((5, 5), np.uint8) @@ -1485,7 +1538,8 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, onset_y=int((img_resized.shape[0]-img_int.shape[0])/2.) #img_resized=np.zeros((int( img_int.shape[0]*(1.8) ) , int( img_int.shape[1]*(2.6) ) )) - #img_resized[ int( img_int.shape[0]*(.4)):int( img_int.shape[0]*(.4))+img_int.shape[0] , int( img_int.shape[1]*(.8)):int( img_int.shape[1]*(.8))+img_int.shape[1] ]=img_int[:,:] + #img_resized[ int( img_int.shape[0]*(.4)):int( img_int.shape[0]*(.4))+img_int.shape[0], + # int( img_int.shape[1]*(.8)):int( img_int.shape[1]*(.8))+img_int.shape[1] ]=img_int[:,:] img_resized[ onset_y:onset_y+img_int.shape[0] , onset_x:onset_x+img_int.shape[1] ]=img_int[:,:] if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]: @@ -1689,14 +1743,18 @@ def do_work_of_slopes_new_curved( mask_biggest2 = cv2.dilate(mask_biggest2, KERNEL, iterations=4) pixel_img = 1 - mask_biggest2 = resize_image(mask_biggest2, int(mask_biggest2.shape[0] * scale_par), int(mask_biggest2.shape[1] * scale_par)) + mask_biggest2 = resize_image(mask_biggest2, + int(mask_biggest2.shape[0] * scale_par), + int(mask_biggest2.shape[1] * scale_par)) cnt_textlines_in_image_ind = return_contours_of_interested_textline(mask_biggest2, pixel_img) try: textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]) except Exception as why: logger.error(why) else: - textlines_cnt_per_region = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text, True) + textlines_cnt_per_region = textline_contours_postprocessing(all_text_region_raw, + slope_for_all, contour_par, + box_text, True) return textlines_cnt_per_region[::-1], box_text, contour, contour_par, crop_coor, index_r_con, slope diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 5f19387..602ad6e 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -370,7 +370,11 @@ def return_textline_contour_with_added_box_coordinate(textline_contour, box_ind return textline_contour -def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, prediction_model, b_s_ocr, num_to_char, textline_light=False, curved_line=False): +def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, + prediction_model, + b_s_ocr, num_to_char, + textline_light=False, + curved_line=False): max_len = 512 padding_token = 299 image_width = 512#max_len * 4 @@ -426,17 +430,23 @@ def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, pr splited_images, splited_images_bin = return_textlines_split_if_needed(img_crop, None) if splited_images: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], + image_height, + image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(1) - img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], + image_height, + image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(-1) else: - img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) + img_fin = preprocess_and_resize_image_for_ocrcnn_model(img_crop, + image_height, + image_width) cropped_lines.append(img_fin) cropped_lines_meging_indexing.append(0) @@ -469,7 +479,12 @@ def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, pr pred_texts_ib = pred_texts[ib].replace("[UNK]", "") extracted_texts.append(pred_texts_ib) - extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] + extracted_texts_merged = [extracted_texts[ind] + if cropped_lines_meging_indexing[ind]==0 + else extracted_texts[ind]+" "+extracted_texts[ind+1] + if cropped_lines_meging_indexing[ind]==1 + else None + for ind in range(len(cropped_lines_meging_indexing))] extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) From b21051db21cf4c0f0e1bbf288cd4e985cc01cb7f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 19:16:00 +0200 Subject: [PATCH 284/374] ProcessPoolExecutor: shutdown during del() instead of atexit() --- src/eynollah/eynollah.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 2e31433..7a28478 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -260,7 +260,6 @@ class Eynollah: # for parallelization of CPU-intensive tasks: self.executor = ProcessPoolExecutor(max_workers=cpu_count()) - atexit.register(self.executor.shutdown) if threshold_art_class_layout: self.threshold_art_class_layout = float(threshold_art_class_layout) @@ -406,6 +405,26 @@ class Eynollah: self.logger.info(f"Model initialization complete ({time.time() - t_start:.1f}s)") + def __del__(self): + if hasattr(self, 'executor') and getattr(self, 'executor'): + self.executor.shutdown() + for model_name in ['model_page', + 'model_classifier', + 'model_bin', + 'model_enhancement', + 'model_region', + 'model_region_1_2', + 'model_region_p2', + 'model_region_fl_np', + 'model_region_fl', + 'model_textline', + 'model_reading_order', + 'model_table', + 'model_ocr', + 'processor']: + if hasattr(self, model_name) and getattr(self, model_name): + delattr(self, model_name) + def cache_images(self, image_filename=None, image_pil=None, dpi=None): ret = {} t_c0 = time.time() From 375e0263d4188ff5ca43037a6176544009c74e17 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 19:16:50 +0200 Subject: [PATCH 285/374] CNN-RNN OCR model: switch to 20250930 version (compatible with TF 2.12 on CPU as well) --- src/eynollah/eynollah.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 7a28478..62ce002 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -327,7 +327,7 @@ class Eynollah: if self.ocr and self.tr: self.model_ocr_dir = dir_models + "/model_eynollah_ocr_trocr_20250919" elif self.ocr and not self.tr: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250904" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250930" if self.tables: if self.light_version: self.model_table_dir = dir_models + "/modelens_table_0t4_201124" @@ -5392,7 +5392,7 @@ class Eynollah_ocr: if self.model_name: self.model_ocr_dir = self.model_name else: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250904" + self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250930" model_ocr = load_model(self.model_ocr_dir , compile=False) self.prediction_model = tf.keras.models.Model( From 61b20cc83d153aa0df2f5b75d6059ac80c730b3c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 19:20:35 +0200 Subject: [PATCH 286/374] tests: switch from subtests to parametrize, use --isolate everywhere to free CUDA memory in between --- Makefile | 2 +- requirements-test.txt | 2 +- tests/test_run.py | 202 ++++++++++++++++++++---------------------- 3 files changed, 100 insertions(+), 106 deletions(-) diff --git a/Makefile b/Makefile index a920615..dd95c0a 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v OCR_MODEL := https://zenodo.org/records/17194824/files/models_ocr_v0_5_0.tar.gz?download=1 -PYTEST_ARGS ?= -vv +PYTEST_ARGS ?= -vv --isolate # BEGIN-EVAL makefile-parser --make-help Makefile diff --git a/requirements-test.txt b/requirements-test.txt index cce9428..3ebcf71 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ pytest -pytest-subtests +pytest-isolate coverage[toml] black diff --git a/tests/test_run.py b/tests/test_run.py index be928a0..59e5099 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -20,23 +20,9 @@ MODELS_LAYOUT = environ.get('MODELS_LAYOUT', str(testdir.joinpath('..', 'models_ MODELS_OCR = environ.get('MODELS_OCR', str(testdir.joinpath('..', 'models_ocr_v0_5_0').resolve())) MODELS_BIN = environ.get('MODELS_BIN', str(testdir.joinpath('..', 'default-2021-03-09').resolve())) -def test_run_eynollah_layout_filename(tmp_path, subtests, pytestconfig, caplog): - infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') - outfile = tmp_path / 'kant_aufklaerung_1784_0020.xml' - args = [ - '-m', MODELS_LAYOUT, - '-i', str(infile), - '-o', str(outfile.parent), - # subtests write to same location - '--overwrite', - ] - if pytestconfig.getoption('verbose') > 0: - args.extend(['-l', 'DEBUG']) - caplog.set_level(logging.INFO) - def only_eynollah(logrec): - return logrec.name == 'eynollah' - runner = CliRunner() - for options in [ +@pytest.mark.parametrize( + "options", + [ [], # defaults ["--allow_scaling", "--curved-line"], ["--allow_scaling", "--curved-line", "--full-layout"], @@ -47,22 +33,34 @@ def test_run_eynollah_layout_filename(tmp_path, subtests, pytestconfig, caplog): # -eoi ... # --do_ocr # --skip_layout_and_reading_order - ]: - with subtests.test(#msg="test CLI", - options=options): - with caplog.filtering(only_eynollah): - result = runner.invoke(layout_cli, args + options, catch_exceptions=False) - assert result.exit_code == 0, result.stdout - logmsgs = [logrec.message for logrec in caplog.records] - assert str(infile) in logmsgs - assert outfile.exists() - tree = page_from_file(str(outfile)).etree - regions = tree.xpath("//page:TextRegion", namespaces=NS) - assert len(regions) >= 2, "result is inaccurate" - regions = tree.xpath("//page:SeparatorRegion", namespaces=NS) - assert len(regions) >= 2, "result is inaccurate" - lines = tree.xpath("//page:TextLine", namespaces=NS) - assert len(lines) == 31, "result is inaccurate" # 29 paragraph lines, 1 page and 1 catch-word line + ], ids=str) +def test_run_eynollah_layout_filename(tmp_path, pytestconfig, caplog, options): + infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') + outfile = tmp_path / 'kant_aufklaerung_1784_0020.xml' + args = [ + '-m', MODELS_LAYOUT, + '-i', str(infile), + '-o', str(outfile.parent), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'eynollah' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(layout_cli, args + options, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + assert str(infile) in logmsgs + assert outfile.exists() + tree = page_from_file(str(outfile)).etree + regions = tree.xpath("//page:TextRegion", namespaces=NS) + assert len(regions) >= 2, "result is inaccurate" + regions = tree.xpath("//page:SeparatorRegion", namespaces=NS) + assert len(regions) >= 2, "result is inaccurate" + lines = tree.xpath("//page:TextLine", namespaces=NS) + assert len(lines) == 31, "result is inaccurate" # 29 paragraph lines, 1 page and 1 catch-word line def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') @@ -86,7 +84,13 @@ def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): assert any(logmsg for logmsg in logmsgs if logmsg.startswith('All jobs done in')) assert len(list(outdir.iterdir())) == 2 -def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, caplog): +@pytest.mark.parametrize( + "options", + [ + [], # defaults + ["--no-patches"], + ], ids=str) +def test_run_eynollah_binarization_filename(tmp_path, pytestconfig, caplog, options): infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') args = [ @@ -100,25 +104,19 @@ def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, ca def only_eynollah(logrec): return logrec.name == 'SbbBinarizer' runner = CliRunner() - for options in [ - [], # defaults - ["--no-patches"], - ]: - with subtests.test(#msg="test CLI", - options=options): - with caplog.filtering(only_eynollah): - result = runner.invoke(binarization_cli, args + options, catch_exceptions=False) - assert result.exit_code == 0, result.stdout - logmsgs = [logrec.message for logrec in caplog.records] - assert any(True for logmsg in logmsgs if logmsg.startswith('Predicting')) - assert outfile.exists() - with Image.open(infile) as original_img: - original_size = original_img.size - with Image.open(outfile) as binarized_img: - binarized_size = binarized_img.size - assert original_size == binarized_size + with caplog.filtering(only_eynollah): + result = runner.invoke(binarization_cli, args + options, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + assert any(True for logmsg in logmsgs if logmsg.startswith('Predicting')) + assert outfile.exists() + with Image.open(infile) as original_img: + original_size = original_img.size + with Image.open(outfile) as binarized_img: + binarized_size = binarized_img.size + assert original_size == binarized_size -def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, caplog): +def test_run_eynollah_binarization_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path args = [ @@ -139,15 +137,19 @@ def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, c assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Predicting')]) == 2 assert len(list(outdir.iterdir())) == 2 -def test_run_eynollah_enhancement_filename(tmp_path, subtests, pytestconfig, caplog): +@pytest.mark.parametrize( + "options", + [ + [], # defaults + ["-sos"], + ], ids=str) +def test_run_eynollah_enhancement_filename(tmp_path, pytestconfig, caplog, options): infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') args = [ '-m', MODELS_LAYOUT, '-i', str(infile), '-o', str(outfile.parent), - # subtests write to same location - '--overwrite', ] if pytestconfig.getoption('verbose') > 0: args.extend(['-l', 'DEBUG']) @@ -155,25 +157,19 @@ def test_run_eynollah_enhancement_filename(tmp_path, subtests, pytestconfig, cap def only_eynollah(logrec): return logrec.name == 'enhancement' runner = CliRunner() - for options in [ - [], # defaults - ["-sos"], - ]: - with subtests.test(#msg="test CLI", - options=options): - with caplog.filtering(only_eynollah): - result = runner.invoke(enhancement_cli, args + options, catch_exceptions=False) - assert result.exit_code == 0, result.stdout - logmsgs = [logrec.message for logrec in caplog.records] - assert any(True for logmsg in logmsgs if logmsg.startswith('Image was enhanced')), logmsgs - assert outfile.exists() - with Image.open(infile) as original_img: - original_size = original_img.size - with Image.open(outfile) as enhanced_img: - enhanced_size = enhanced_img.size - assert (original_size == enhanced_size) == ("-sos" in options) + with caplog.filtering(only_eynollah): + result = runner.invoke(enhancement_cli, args + options, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + assert any(True for logmsg in logmsgs if logmsg.startswith('Image was enhanced')), logmsgs + assert outfile.exists() + with Image.open(infile) as original_img: + original_size = original_img.size + with Image.open(outfile) as enhanced_img: + enhanced_size = enhanced_img.size + assert (original_size == enhanced_size) == ("-sos" in options) -def test_run_eynollah_enhancement_directory(tmp_path, subtests, pytestconfig, caplog): +def test_run_eynollah_enhancement_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path args = [ @@ -194,7 +190,7 @@ def test_run_eynollah_enhancement_directory(tmp_path, subtests, pytestconfig, ca assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Image was enhanced')]) == 2 assert len(list(outdir.iterdir())) == 2 -def test_run_eynollah_mbreorder_filename(tmp_path, subtests, pytestconfig, caplog): +def test_run_eynollah_mbreorder_filename(tmp_path, pytestconfig, caplog): infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.xml') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.xml') args = [ @@ -223,7 +219,7 @@ def test_run_eynollah_mbreorder_filename(tmp_path, subtests, pytestconfig, caplo #assert in_order != out_order assert out_order == ['r_1_1', 'r_2_1', 'r_2_2', 'r_2_3'] -def test_run_eynollah_mbreorder_directory(tmp_path, subtests, pytestconfig, caplog): +def test_run_eynollah_mbreorder_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path args = [ @@ -245,7 +241,15 @@ def test_run_eynollah_mbreorder_directory(tmp_path, subtests, pytestconfig, capl #assert len([logmsg for logmsg in logmsgs if logmsg.startswith('???')]) == 2 assert len(list(outdir.iterdir())) == 2 -def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): +@pytest.mark.parametrize( + "options", + [ + [], # defaults + ["-doit", #str(outrenderfile.parent)], + ], + ["-trocr"], + ], ids=str) +def test_run_eynollah_ocr_filename(tmp_path, pytestconfig, caplog, options): infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.xml') outrenderfile = tmp_path.joinpath('render').joinpath('kant_aufklaerung_1784_0020.png') @@ -255,8 +259,6 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): '-i', str(infile), '-dx', str(infile.parent), '-o', str(outfile.parent), - # subtests write to same location - '--overwrite', ] if pytestconfig.getoption('verbose') > 0: args.extend(['-l', 'DEBUG']) @@ -264,33 +266,25 @@ def test_run_eynollah_ocr_filename(tmp_path, subtests, pytestconfig, caplog): def only_eynollah(logrec): return logrec.name == 'eynollah' runner = CliRunner() - for options in [ - # kba Fri Sep 26 12:53:49 CEST 2025 - # Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged - # [], # defaults - # ["-doit", str(outrenderfile.parent)], - ["-trocr"], - ]: - with subtests.test(#msg="test CLI", - options=options): - with caplog.filtering(only_eynollah): - result = runner.invoke(ocr_cli, args + options, catch_exceptions=False) - assert result.exit_code == 0, result.stdout - logmsgs = [logrec.message for logrec in caplog.records] - # FIXME: ocr has no logging! - #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs - assert outfile.exists() - if "-doit" in options: - assert outrenderfile.exists() - #in_tree = page_from_file(str(infile)).etree - #in_order = in_tree.xpath("//page:OrderedGroup//@regionRef", namespaces=NS) - out_tree = page_from_file(str(outfile)).etree - out_texts = out_tree.xpath("//page:TextLine/page:TextEquiv[last()]/page:Unicode/text()", namespaces=NS) - assert len(out_texts) >= 2, ("result is inaccurate", out_texts) - assert sum(map(len, out_texts)) > 100, ("result is inaccurate", out_texts) + if "-doit" in options: + options.insert(options.index("-doit") + 1, str(outrenderfile.parent)) + with caplog.filtering(only_eynollah): + result = runner.invoke(ocr_cli, args + options, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + # FIXME: ocr has no logging! + #assert any(True for logmsg in logmsgs if logmsg.startswith('???')), logmsgs + assert outfile.exists() + if "-doit" in options: + assert outrenderfile.exists() + #in_tree = page_from_file(str(infile)).etree + #in_order = in_tree.xpath("//page:OrderedGroup//@regionRef", namespaces=NS) + out_tree = page_from_file(str(outfile)).etree + out_texts = out_tree.xpath("//page:TextLine/page:TextEquiv[last()]/page:Unicode/text()", namespaces=NS) + assert len(out_texts) >= 2, ("result is inaccurate", out_texts) + assert sum(map(len, out_texts)) > 100, ("result is inaccurate", out_texts) -@pytest.mark.skip("Disabled until NHWC/NCHW error in https://github.com/qurator-spk/eynollah/actions/runs/18019655200/job/51273541895 debugged") -def test_run_eynollah_ocr_directory(tmp_path, subtests, pytestconfig, caplog): +def test_run_eynollah_ocr_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path args = [ From a3d8197930b9e2c07862186d23ee192dc0347ff4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 21:50:21 +0200 Subject: [PATCH 287/374] makefile: update model URL --- Makefile | 50 ++++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index dd95c0a..357aa47 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,16 @@ DOCKER ?= docker #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.0/models_eynollah.tar.gz #SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz SEG_MODEL := https://zenodo.org/records/17194824/files/models_layout_v0_5_0.tar.gz?download=1 +SEG_MODELFILE = $(notdir $(patsubst %?download=1,%,$(SEG_MODEL))) +SEG_MODELNAME = $(SEG_MODELFILE:%.tar.gz=%) BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip +BIN_MODELFILE = $(notdir $(BIN_MODEL)) +BIN_MODELNAME := default-2021-03-09 -OCR_MODEL := https://zenodo.org/records/17194824/files/models_ocr_v0_5_0.tar.gz?download=1 +OCR_MODEL := https://zenodo.org/records/17236998/files/models_ocr_v0_5_1.tar.gz?download=1 +OCR_MODELFILE = $(notdir $(patsubst %?download=1,%,$(OCR_MODEL))) +OCR_MODELNAME = $(OCR_MODELFILE:%.tar.gz=%) PYTEST_ARGS ?= -vv --isolate @@ -31,7 +37,8 @@ help: @echo " install Install package with pip" @echo " install-dev Install editable with pip" @echo " deps-test Install test dependencies with pip" - @echo " models Download and extract models to $(CURDIR)/models_layout_v0_5_0" + @echo " models Download and extract models to $(CURDIR):" + @echo " $(BIN_MODELNAME) $(SEG_MODELNAME) $(OCR_MODELNAME)" @echo " smoke-test Run simple CLI check" @echo " ocrd-test Run OCR-D CLI check" @echo " test Run unit tests" @@ -42,33 +49,29 @@ help: @echo " PYTEST_ARGS pytest args for 'test' (Set to '-s' to see log output during test execution, '-vv' to see individual tests. [$(PYTEST_ARGS)]" @echo " SEG_MODEL URL of 'models' archive to download for segmentation 'test' [$(SEG_MODEL)]" @echo " BIN_MODEL URL of 'models' archive to download for binarization 'test' [$(BIN_MODEL)]" + @echo " OCR_MODEL URL of 'models' archive to download for binarization 'test' [$(OCR_MODEL)]" @echo "" # END-EVAL # Download and extract models to $(PWD)/models_layout_v0_5_0 -models: models_layout_v0_5_0 models_ocr_v0_5_0 default-2021-03-09 +models: $(BIN_MODELNAME) $(SEG_MODELNAME) $(OCR_MODELNAME) -models_layout_v0_5_0: models_layout_v0_5_0.tar.gz - tar zxf models_layout_v0_5_0.tar.gz - -models_layout_v0_5_0.tar.gz: +$(BIN_MODELFILE): + wget -O $@ $(BIN_MODEL) +$(SEG_MODELFILE): wget -O $@ $(SEG_MODEL) - -models_ocr_v0_5_0: models_ocr_v0_5_0.tar.gz - tar zxf models_ocr_v0_5_0.tar.gz - -models_ocr_v0_5_0.tar.gz: +$(OCR_MODELFILE): wget -O $@ $(OCR_MODEL) -default-2021-03-09: $(notdir $(BIN_MODEL)) - unzip $(notdir $(BIN_MODEL)) +$(BIN_MODELNAME): $(BIN_MODELFILE) mkdir $@ - mv $(basename $(notdir $(BIN_MODEL))) $@ - -$(notdir $(BIN_MODEL)): - wget $(BIN_MODEL) + unzip -d $@ $< +$(SEG_MODELNAME): $(SEG_MODELFILE) + tar zxf $< +$(OCR_MODELNAME): $(OCR_MODELFILE) + tar zxf $< build: $(PIP) install build @@ -82,7 +85,10 @@ install: install-dev: $(PIP) install -e .$(and $(EXTRAS),[$(EXTRAS)]) -deps-test: models_layout_v0_5_0 +ifeq (OCR,$(findstring OCR, $(EXTRAS))) +deps-test: $(OCR_MODELNAME) +endif +deps-test: $(BIN_MODELNAME) $(SEG_MODELNAME) $(PIP) install -r requirements-test.txt smoke-test: TMPDIR != mktemp -d @@ -123,9 +129,9 @@ ocrd-test: tests/resources/kant_aufklaerung_1784_0020.tif $(RM) -r $(TMPDIR) # Run unit tests -test: export MODELS_LAYOUT=$(CURDIR)/models_layout_v0_5_0 -test: export MODELS_OCR=$(CURDIR)/models_ocr_v0_5_0 -test: export MODELS_BIN=$(CURDIR)/default-2021-03-09 +test: export MODELS_LAYOUT=$(CURDIR)/$(SEG_MODELNAME) +test: export MODELS_OCR=$(CURDIR)/$(OCR_MODELNAME) +test: export MODELS_BIN=$(CURDIR)/$(BIN_MODELNAME) test: $(PYTHON) -m pytest tests --durations=0 --continue-on-collection-errors $(PYTEST_ARGS) From c86e59f481ee47ccb9938b7f6105f95f626c5f17 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 22:03:46 +0200 Subject: [PATCH 288/374] CI: update model key, split up cache restore/save --- .github/workflows/test-eynollah.yml | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index 042e508..ca213cb 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -24,17 +24,17 @@ jobs: sudo rm -rf "$AGENT_TOOLSDIRECTORY" df -h - uses: actions/checkout@v4 - - uses: actions/cache@v4 + - uses: actions/cache/restore@v4 id: seg_model_cache with: path: models_layout_v0_5_0 - key: ${{ runner.os }}-models - - uses: actions/cache@v4 + key: ${{ runner.os }}-seg-models + - uses: actions/cache/restore@v4 id: ocr_model_cache with: - path: models_ocr_v0_5_0 - key: ${{ runner.os }}-models - - uses: actions/cache@v4 + path: models_ocr_v0_5_1 + key: ${{ runner.os }}-ocr-models + - uses: actions/cache/restore@v4 id: bin_model_cache with: path: default-2021-03-09 @@ -42,6 +42,21 @@ jobs: - name: Download models if: steps.seg_model_cache.outputs.cache-hit != 'true' || steps.bin_model_cache.outputs.cache-hit != 'true' || steps.ocr_model_cache.outputs.cache-hit != true run: make models + - uses: actions/cache/save@v4 + if: steps.seg_model_cache.outputs.cache-hit != 'true' + with: + path: models_layout_v0_5_0 + key: ${{ runner.os }}-seg-models + - uses: actions/cache/save@v4 + if: steps.ocr_model_cache.outputs.cache-hit != 'true' + with: + path: models_ocr_v0_5_1 + key: ${{ runner.os }}-ocr-models + - uses: actions/cache/save@v4 + if: steps.bin_model_cache.outputs.cache-hit != 'true' + with: + path: default-2021-03-09 + key: ${{ runner.os }}-modelbin - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From ad129ed46c70b03fea7b48060e40e2451b40b975 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 22:05:53 +0200 Subject: [PATCH 289/374] CI: remove OS from model cache keys --- .github/workflows/test-eynollah.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index ca213cb..9d5b2c8 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -28,17 +28,17 @@ jobs: id: seg_model_cache with: path: models_layout_v0_5_0 - key: ${{ runner.os }}-seg-models + key: seg-models - uses: actions/cache/restore@v4 id: ocr_model_cache with: path: models_ocr_v0_5_1 - key: ${{ runner.os }}-ocr-models + key: ocr-models - uses: actions/cache/restore@v4 id: bin_model_cache with: path: default-2021-03-09 - key: ${{ runner.os }}-modelbin + key: bin-models - name: Download models if: steps.seg_model_cache.outputs.cache-hit != 'true' || steps.bin_model_cache.outputs.cache-hit != 'true' || steps.ocr_model_cache.outputs.cache-hit != true run: make models @@ -46,17 +46,17 @@ jobs: if: steps.seg_model_cache.outputs.cache-hit != 'true' with: path: models_layout_v0_5_0 - key: ${{ runner.os }}-seg-models + key: seg-models - uses: actions/cache/save@v4 if: steps.ocr_model_cache.outputs.cache-hit != 'true' with: path: models_ocr_v0_5_1 - key: ${{ runner.os }}-ocr-models + key: ocr-models - uses: actions/cache/save@v4 if: steps.bin_model_cache.outputs.cache-hit != 'true' with: path: default-2021-03-09 - key: ${{ runner.os }}-modelbin + key: bin-models - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: From 7daec392b9846931b932d48fde71680ab4bf33e9 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 22:10:45 +0200 Subject: [PATCH 290/374] Dockerfile: fix up CUDA installation for mixed TF/Torch --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4ba498b..a15776e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,8 @@ RUN ocrd ocrd-tool ocrd-tool.json dump-tools > $(dirname $(ocrd bashlib filename RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib filename))/ocrd-all-module-dir.json # install everything and reduce image size RUN make install EXTRAS=OCR && rm -rf /build/eynollah +# fixup for broken cuDNN installation (Torch pulls in 8.5.0, which is incompatible with Tensorflow) +RUN pip install nvidia-cudnn-cu11==8.6.0.163 # smoke test RUN eynollah --help From f0de1adabf45f3dd70df72ddc09795a4512d5316 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 23:12:18 +0200 Subject: [PATCH 291/374] rm loky dependency --- .gitignore | 4 ++++ requirements.txt | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0d5d834..3cc0eac 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,11 @@ __pycache__ sbb_newspapers_org_image/pylint.log models_eynollah* +models_ocr* +models_layout* +default-2021-03-09 output.html /build /dist *.tif +TAGS diff --git a/requirements.txt b/requirements.txt index 4bc0c6a..db1d7df 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,5 +5,4 @@ scikit-learn >= 0.23.2 tensorflow < 2.13 numba <= 0.58.1 scikit-image -loky biopython From 3aa7ad04fafd842fe31c36094a2b51fa43cc1bd3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 30 Sep 2025 23:14:52 +0200 Subject: [PATCH 292/374] :memo: update changelog --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad9a09..f6776d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,33 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Fixed: + + * :fire: polygons: avoid invalid paths (use `Polygon.buffer()` instead of dilation etc.) + * `return_boxes_of_images_by_order_of_reading_new`: avoid Numpy.dtype mismatch, simplify + * `return_boxes_of_images_by_order_of_reading_new`: log any exceptions instead of ignoring + * `filter_contours_without_textline_inside`: avoid removing from duplicate lists twice + * `get_marginals`: exit early if no peaks found to avoid spurious overlap mask + * `get_smallest_skew`: after shifting search range of rotation angle, use overall best result + * Dockerfile: fix CUDA installation (cuDNN contested between Torch and TF due to extra OCR) + * OCR: re-instate missing methods and fix `utils_ocr` function calls + * :fire: writer: `SeparatorRegion` needs `SeparatorRegionType` (not `ImageRegionType`) +f458e3e + * tests: switch from `pytest-subtests` to `parametrize` so we can use `pytest-isolate` + (so CUDA memory gets freed between tests if running on GPU) + +Changed: + + * polygons: slightly widen for regions and lines, increase for separators + * various refactorings, some code style and identifier improvements + * deskewing/multiprocessing: switch back to ProcessPoolExecutor (faster), + but use shared memory if necessary, and switch back from `loky` to stdlib, + and shutdown in `del()` instead of `atexit` + * :fire: OCR: switch CNN-RNN model to `20250930` version compatible with TF 2.12 on CPU, too + * :fire: writer: use `@type='heading'` instead of `'header'` for headings + * CI: update+improve model caching + + ## [0.5.0] - 2025-09-26 Fixed: From 558867eb245d7db1e7a9780d21d226d5729a3c96 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 00:04:07 +0200 Subject: [PATCH 293/374] fix typo --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ad9a09..8c6c000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ Added: Fixed: * allow empty imports for optional dependencies - * avoid Numpy warnings (empty slices etc) + * avoid Numpy warnings (empty slices etc.) * remove deprecated Numpy types * binarization CLI: make `dir_in` usable again From 9ce127eb51997f6779f6d9877e4eb506ed5fda21 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 00:04:53 +0200 Subject: [PATCH 294/374] remove unnecessary backslash --- src/eynollah/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 6eeabd0..58592bd 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1452,7 +1452,7 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, gray = cv2.bitwise_not(separators_closeup_n_binary) gray=gray.astype(np.uint8) - bw = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, \ + bw = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 15, -2) horizontal = np.copy(bw) vertical = np.copy(bw) From 1d0616eb6918d6017e258fb50356eef0fefd685a Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 00:15:11 +0200 Subject: [PATCH 295/374] comparisons to None should not use the equality operators --- src/eynollah/utils/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 58592bd..152ac6e 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1211,7 +1211,7 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): ##plt.plot(z) ##plt.show() - if contours_main != None: + if contours_main is not None: areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] @@ -1222,7 +1222,7 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) - if len(contours_header) != None: + if len(contours_header) is not None: areas_header = np.array([cv2.contourArea(contours_header[j]) for j in range(len(contours_header))]) M_header = [cv2.moments(contours_header[j]) for j in range(len(contours_header))] cx_header = [(M_header[j]["m10"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] @@ -1243,9 +1243,9 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): if len(cy_main) > 0 and np.max(cy_main) > np.max(peaks_neg_new): cy_main = np.array(cy_main) * (np.max(peaks_neg_new) / np.max(cy_main)) - 10 - if contours_main != None: + if contours_main is not None: indexer_main = np.arange(len(contours_main)) - if contours_main != None: + if contours_main is not None: len_main = len(contours_main) else: len_main = 0 From 70af00182b2332f33e7872b6abc1af9bbba787bc Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 00:20:18 +0200 Subject: [PATCH 296/374] mutable defaults are the source of all evil --- train/models.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/train/models.py b/train/models.py index 8841bd3..fdc5437 100644 --- a/train/models.py +++ b/train/models.py @@ -394,7 +394,9 @@ def resnet50_unet(n_classes, input_height=224, input_width=224, task="segmentati return model -def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=[128, 64], transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): +def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=None, transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): + if mlp_head_units is None: + mlp_head_units = [128, 64] inputs = layers.Input(shape=(input_height, input_width, 3)) #transformer_units = [ @@ -516,7 +518,9 @@ def vit_resnet50_unet(n_classes, patch_size_x, patch_size_y, num_patches, mlp_he return model -def vit_resnet50_unet_transformer_before_cnn(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=[128, 64], transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): +def vit_resnet50_unet_transformer_before_cnn(n_classes, patch_size_x, patch_size_y, num_patches, mlp_head_units=None, transformer_layers=8, num_heads =4, projection_dim = 64, input_height=224, input_width=224, task="segmentation", weight_decay=1e-6, pretraining=False): + if mlp_head_units is None: + mlp_head_units = [128, 64] inputs = layers.Input(shape=(input_height, input_width, 3)) ##transformer_units = [ From f2f93e0251de3421b26d9b9f18c7d581846e82af Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 00:26:27 +0200 Subject: [PATCH 297/374] list literal is faster than using list constructor to create a new list --- src/eynollah/utils/__init__.py | 21 +++++++-------------- src/eynollah/utils/contour.py | 3 +-- src/eynollah/utils/separate_lines.py | 6 ++---- train/inference.py | 3 +-- train/train.py | 3 +-- 5 files changed, 12 insertions(+), 24 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 152ac6e..7c06900 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -138,8 +138,7 @@ def return_x_start_end_mothers_childs_and_type_of_reading_order( min_ys=np.min(y_sep) max_ys=np.max(y_sep) - y_mains=[] - y_mains.append(min_ys) + y_mains= [min_ys] y_mains_sep_ohne_grenzen=[] for ii in range(len(new_main_sep_y)): @@ -493,8 +492,7 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl # print(forest[np.argmin(z[forest]) ] ) if not isNaN(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) - forest = [] - forest.append(peaks_neg_fin[i + 1]) + forest = [peaks_neg_fin[i + 1]] if i == (len(peaks_neg_fin) - 1): # print(print(forest[np.argmin(z[forest]) ] )) if not isNaN(forest[np.argmin(z[forest])]): @@ -662,8 +660,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): # print(forest[np.argmin(z[forest]) ] ) if not isNaN(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) - forest = [] - forest.append(peaks_neg_fin[i + 1]) + forest = [peaks_neg_fin[i + 1]] if i == (len(peaks_neg_fin) - 1): # print(print(forest[np.argmin(z[forest]) ] )) if not isNaN(forest[np.argmin(z[forest])]): @@ -1235,8 +1232,7 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): y_max_header = np.array([np.max(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) # print(cy_main,'mainy') - peaks_neg_new = [] - peaks_neg_new.append(0 + y_ref) + peaks_neg_new = [0 + y_ref] for iii in range(len(peaks_neg)): peaks_neg_new.append(peaks_neg[iii] + y_ref) peaks_neg_new.append(textline_mask.shape[0] + y_ref) @@ -1404,8 +1400,7 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( return img_p_in[:,:,0], special_separators def return_points_with_boundies(peaks_neg_fin, first_point, last_point): - peaks_neg_tot = [] - peaks_neg_tot.append(first_point) + peaks_neg_tot = [first_point] for ii in range(len(peaks_neg_fin)): peaks_neg_tot.append(peaks_neg_fin[ii]) peaks_neg_tot.append(last_point) @@ -1588,8 +1583,7 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, args_cy_splitter=np.argsort(cy_main_splitters) cy_main_splitters_sort=cy_main_splitters[args_cy_splitter] - splitter_y_new=[] - splitter_y_new.append(0) + splitter_y_new= [0] for i in range(len(cy_main_splitters_sort)): splitter_y_new.append( cy_main_splitters_sort[i] ) splitter_y_new.append(region_pre_p.shape[0]) @@ -1663,8 +1657,7 @@ def return_boxes_of_images_by_order_of_reading_new( num_col, peaks_neg_fin = find_num_col( regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], num_col_classifier, tables, multiplier=3.) - peaks_neg_fin_early=[] - peaks_neg_fin_early.append(0) + peaks_neg_fin_early= [0] #print(peaks_neg_fin,'peaks_neg_fin') for p_n in peaks_neg_fin: peaks_neg_fin_early.append(p_n) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 0e84153..0be8879 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -239,8 +239,7 @@ def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) if len(cont_int)==0: - cont_int = [] - cont_int.append(contour_par) + cont_int = [contour_par] confidence_contour = 0 else: cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index ead5cfb..c87653c 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1174,8 +1174,7 @@ def separate_lines_new_inside_tiles(img_path, thetha): if diff_peaks[i] > cut_off: if not np.isnan(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) - forest = [] - forest.append(peaks_neg[i + 1]) + forest = [peaks_neg[i + 1]] if i == (len(peaks_neg) - 1): if not np.isnan(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) @@ -1195,8 +1194,7 @@ def separate_lines_new_inside_tiles(img_path, thetha): if diff_peaks_pos[i] > cut_off: if not np.isnan(forest[np.argmax(z[forest])]): peaks_pos_true.append(forest[np.argmax(z[forest])]) - forest = [] - forest.append(peaks[i + 1]) + forest = [peaks[i + 1]] if i == (len(peaks) - 1): if not np.isnan(forest[np.argmax(z[forest])]): peaks_pos_true.append(forest[np.argmax(z[forest])]) diff --git a/train/inference.py b/train/inference.py index 094c528..0e55aa8 100644 --- a/train/inference.py +++ b/train/inference.py @@ -305,8 +305,7 @@ class sbb_predict: input_1= np.zeros( (inference_bs, img_height, img_width,3)) - starting_list_of_regions = [] - starting_list_of_regions.append( list(range(labels_con.shape[2])) ) + starting_list_of_regions = [list(range(labels_con.shape[2]))] index_update = 0 index_selected = starting_list_of_regions[0] diff --git a/train/train.py b/train/train.py index e8e92af..795009a 100644 --- a/train/train.py +++ b/train/train.py @@ -365,8 +365,7 @@ def run(_config, n_classes, n_epochs, input_height, y_tot=np.zeros((testX.shape[0],n_classes)) - score_best=[] - score_best.append(0) + score_best= [0] num_rows = return_number_of_total_training_data(dir_train) weights=[] From 91d2a74ac950e55e75c0c03ece817ae96a4fc377 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 00:38:01 +0200 Subject: [PATCH 298/374] remove redundant parentheses --- src/eynollah/eynollah.py | 4 ++-- src/eynollah/plot.py | 2 +- src/eynollah/utils/__init__.py | 8 ++++---- src/eynollah/utils/counter.py | 2 +- src/eynollah/utils/marginals.py | 2 +- src/eynollah/utils/separate_lines.py | 14 +++++++------- src/eynollah/writer.py | 2 +- train/inference.py | 8 ++++---- train/train.py | 10 +++++----- train/utils.py | 6 +++--- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 20954a0..63f7005 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4886,9 +4886,9 @@ class Eynollah: textline_mask_tot_ea_org[img_revised_tab==drop_label_in_full_layout] = 0 - text_only = ((img_revised_tab[:, :] == 1)) * 1 + text_only = (img_revised_tab[:, :] == 1) * 1 if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - text_only_d = ((text_regions_p_1_n[:, :] == 1)) * 1 + text_only_d = (text_regions_p_1_n[:, :] == 1) * 1 #print("text region early 2 in %.1fs", time.time() - t0) ###min_con_area = 0.000005 diff --git a/src/eynollah/plot.py b/src/eynollah/plot.py index 412ae5a..c026e94 100644 --- a/src/eynollah/plot.py +++ b/src/eynollah/plot.py @@ -12,7 +12,7 @@ from .utils import crop_image_inside_box from .utils.rotate import rotate_image_different from .utils.resize import resize_image -class EynollahPlotter(): +class EynollahPlotter: """ Class collecting all the plotting and image writing methods """ diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 7c06900..de083f5 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1267,11 +1267,11 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): top = peaks_neg_new[i] down = peaks_neg_new[i + 1] indexes_in = matrix_of_orders[:, 0][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] + (matrix_of_orders[:, 3] < down)] cxs_in = matrix_of_orders[:, 2][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] + (matrix_of_orders[:, 3] < down)] cys_in = matrix_of_orders[:, 3][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] + (matrix_of_orders[:, 3] < down)] types_of_text = matrix_of_orders[:, 1][(matrix_of_orders[:, 3] >= top) & (matrix_of_orders[:, 3] < down)] index_types_of_text = matrix_of_orders[:, 4][(matrix_of_orders[:, 3] >= top) & @@ -1408,7 +1408,7 @@ def return_points_with_boundies(peaks_neg_fin, first_point, last_point): def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, pixel_lines, contours_h=None): t_ins_c0 = time.time() - separators_closeup=( (region_pre_p[:,:,:]==pixel_lines))*1 + separators_closeup= (region_pre_p[:, :, :] == pixel_lines) * 1 separators_closeup[0:110,:,:]=0 separators_closeup[separators_closeup.shape[0]-150:,:,:]=0 diff --git a/src/eynollah/utils/counter.py b/src/eynollah/utils/counter.py index 9a3ed70..e6205c8 100644 --- a/src/eynollah/utils/counter.py +++ b/src/eynollah/utils/counter.py @@ -3,7 +3,7 @@ from collections import Counter REGION_ID_TEMPLATE = 'region_%04d' LINE_ID_TEMPLATE = 'region_%04d_line_%04d' -class EynollahIdCounter(): +class EynollahIdCounter: def __init__(self, region_idx=0, line_idx=0): self._counter = Counter() diff --git a/src/eynollah/utils/marginals.py b/src/eynollah/utils/marginals.py index ac8dc1d..9ec0737 100644 --- a/src/eynollah/utils/marginals.py +++ b/src/eynollah/utils/marginals.py @@ -76,7 +76,7 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve peaks, _ = find_peaks(text_with_lines_y_rev, height=0) peaks=np.array(peaks) - peaks=peaks[(peaks>first_nonzero) & ((peaksfirst_nonzero) & (peaks < last_nonzero)] peaks=peaks[region_sum_0[peaks]=batchsize: ret_x = ret_x/255. - yield (ret_x, ret_y) + yield ret_x, ret_y ret_x= np.zeros((batchsize, height,width, 3)).astype(np.int16) ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) batchcount = 0 @@ -446,7 +446,7 @@ def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batch ret_y[batchcount, :] = label_class batchcount+=1 if batchcount>=batchsize: - yield (ret_x, ret_y) + yield ret_x, ret_y ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) batchcount = 0 @@ -464,7 +464,7 @@ def generate_arrays_from_folder_reading_order(classes_file_dir, modal_dir, batch ret_y[batchcount, :] = label_class batchcount+=1 if batchcount>=batchsize: - yield (ret_x, ret_y) + yield ret_x, ret_y ret_x= np.zeros((batchsize, height, width, 3))#.astype(np.int16) ret_y= np.zeros((batchsize, n_classes)).astype(np.int16) batchcount = 0 From e027bc038e28736a5557d342f0adcbd153bacc57 Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:05:15 +0200 Subject: [PATCH 299/374] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9dc4824..144ccd4 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ make install EXTRAS=OCR Pretrained models can be downloaded from [zenodo](https://zenodo.org/records/17194824) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). -For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). +For documentation on models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). Model cards are also provided for our trained models. ## Training @@ -74,7 +74,7 @@ image enhancement, text recognition (OCR), and reading order detection. ### Layout Analysis The layout analysis module is responsible for detecting layout elements, identifying text lines, and determining reading -order using either heuristic methods or a reading order detection model. +order using either heuristic methods or a [pretrained reading order detection model](https://github.com/qurator-spk/eynollah#machine-based-reading-order). Reading order detection can be performed either as part of layout analysis based on image input, or, currently under development, based on pre-existing layout analysis results in PAGE-XML format as input. @@ -174,6 +174,7 @@ If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynol (because some other preprocessing step was in effect like `denoised`), then the output PAGE-XML will be based on that as new top-level (`@imageFilename`) + ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_5_0 In general, it makes more sense to add other workflow steps **after** Eynollah. From 4514d417a77a61d6143622d3503ea475106cb25b Mon Sep 17 00:00:00 2001 From: cneud <952378+cneud@users.noreply.github.com> Date: Wed, 1 Oct 2025 01:16:25 +0200 Subject: [PATCH 300/374] force GH markdown code block in list --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 144ccd4..3ba5086 100644 --- a/README.md +++ b/README.md @@ -174,8 +174,7 @@ If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynol (because some other preprocessing step was in effect like `denoised`), then the output PAGE-XML will be based on that as new top-level (`@imageFilename`) - - ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_5_0 + ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models eynollah_layout_v0_5_0 In general, it makes more sense to add other workflow steps **after** Eynollah. From 5725e4fd1f6bab4c1152c88cc28c44c0e8c2c584 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Wed, 1 Oct 2025 15:58:03 +0200 Subject: [PATCH 301/374] =?UTF-8?q?-Continue=20processing=20when=20num=5Fc?= =?UTF-8?q?ol=20is=20None=20but=20textregions=20exist.=20-Convert=20margin?= =?UTF-8?q?al-only=20=20to=20main=20body=20if=20no=20main=20body=20is=20pr?= =?UTF-8?q?esent.=20-Reset=20deskew=20angle=20to=200=20when=20text=20regio?= =?UTF-8?q?n=20density=20(textregion=20area=20to=20page=20area)=20<=200.3?= =?UTF-8?q?=20and=20angle=20>=2045=C2=B0.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/eynollah/eynollah.py | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 20954a0..5e8412e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -1,4 +1,4 @@ -# pylint: disable=no-member,invalid-name,line-too-long,missing-function-docstring,missing-class-docstring,too-many-branches +#run_single# pylint: disable=no-member,invalid-name,line-too-long,missing-function-docstring,missing-class-docstring,too-many-branches # pylint: disable=too-many-locals,wrong-import-position,too-many-lines,too-many-statements,chained-comparison,fixme,broad-except,c-extension-no-member # pylint: disable=too-many-public-methods,too-many-arguments,too-many-instance-attributes,too-many-public-methods, # pylint: disable=consider-using-enumerate @@ -2245,6 +2245,7 @@ class Eynollah: ##mask_texts_only = cv2.dilate(mask_texts_only, KERNEL, iterations=1) mask_texts_only = cv2.dilate(mask_texts_only, kernel=np.ones((2,2), np.uint8), iterations=1) + mask_images_only=(prediction_regions_org[:,:] ==2)*1 polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) @@ -2280,20 +2281,18 @@ class Eynollah: text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_texts, color=(1,1,1)) - #plt.imshow(textline_mask_tot_ea) #plt.show() textline_mask_tot_ea[(text_regions_p_true==0) | (text_regions_p_true==4) ] = 0 - #plt.imshow(textline_mask_tot_ea) #plt.show() #print("inside 4 ", time.time()-t_in) self.logger.debug("exit get_regions_light_v") - return text_regions_p_true, erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin, confidence_matrix + return text_regions_p_true, erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin, confidence_matrix, polygons_of_only_texts else: img_bin = resize_image(img_bin,img_height_h, img_width_h ) self.logger.debug("exit get_regions_light_v") - return None, erosion_hurts, None, textline_mask_tot_ea, img_bin, None + return None, erosion_hurts, None, textline_mask_tot_ea, img_bin, None, None def get_regions_from_xy_2models(self,img,is_image_enhanced, num_col_classifier): self.logger.debug("enter get_regions_from_xy_2models") @@ -2386,7 +2385,7 @@ class Eynollah: text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) self.logger.debug("exit get_regions_from_xy_2models") - return text_regions_p_true, erosion_hurts, polygons_lines_xml + return text_regions_p_true, erosion_hurts, polygons_lines_xml, polygons_of_only_texts except: if self.input_binary: prediction_bin = np.copy(img_org) @@ -2436,7 +2435,7 @@ class Eynollah: erosion_hurts = True self.logger.debug("exit get_regions_from_xy_2models") - return text_regions_p_true, erosion_hurts, polygons_lines_xml + return text_regions_p_true, erosion_hurts, polygons_lines_xml, polygons_of_only_texts def do_order_of_regions_full_layout( self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): @@ -4701,7 +4700,7 @@ class Eynollah: self.logger.info("Step 2/5: Basic Processing Mode") self.logger.info("Skipping layout analysis and reading order detection") - _ ,_, _, textline_mask_tot_ea, img_bin_light, _ = \ + _ ,_, _, textline_mask_tot_ea, img_bin_light, _,_= \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier, skip_layout_and_reading_order=self.skip_layout_and_reading_order) @@ -4768,10 +4767,10 @@ class Eynollah: if self.light_version: self.logger.info("Using light version processing") - text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ + text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light, confidence_matrix, polygons_text_early = \ self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier) #print("text region early -2 in %.1fs", time.time() - t0) - + if num_col_classifier == 1 or num_col_classifier ==2: if num_col_classifier == 1: img_w_new = 1000 @@ -4793,9 +4792,9 @@ class Eynollah: #self.logger.info("run graphics %.1fs ", time.time() - t1t) #print("text region early -3 in %.1fs", time.time() - t0) textline_mask_tot_ea_org = np.copy(textline_mask_tot_ea) - #print("text region early -4 in %.1fs", time.time() - t0) + else: - text_regions_p_1 ,erosion_hurts, polygons_lines_xml = \ + text_regions_p_1 ,erosion_hurts, polygons_lines_xml, polygons_text_early = \ self.get_regions_from_xy_2models(img_res, is_image_enhanced, num_col_classifier) self.logger.info(f"Textregion detection took {time.time() - t1:.1f}s") @@ -4811,7 +4810,7 @@ class Eynollah: #plt.show() self.logger.info(f"Layout analysis complete ({time.time() - t1:.1f}s)") - if not num_col: + if not num_col and len(polygons_text_early) == 0: self.logger.info("No columns detected - generating empty PAGE-XML") pcgts = self.writer.build_pagexml_no_full_layout( @@ -4848,6 +4847,15 @@ class Eynollah: textline_mask_tot, text_regions_p, image_page_rotated = \ self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction) + + + if image_page.shape[0]!=0 and image_page.shape[1]!=0: + # if ratio of text regions to page area is smaller that 0.3, deskew angle is not aloowed to exceed 45 + if ( ( text_regions_p[:,:]==1).sum() + (text_regions_p[:,:]==4).sum() ) / float(image_page.shape[0]*image_page.shape[1] ) <= 0.3 and abs(slope_deskew) > 45: + slope_deskew = 0 + + if (text_regions_p[:,:]==1).sum() == 0: + text_regions_p[:,:][text_regions_p[:,:]==4] = 1 self.logger.info("Step 3/5: Text Line Detection") @@ -4894,6 +4902,8 @@ class Eynollah: ###min_con_area = 0.000005 contours_only_text, hir_on_text = return_contours_of_image(text_only) contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) + + if len(contours_only_text_parent) > 0: areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) @@ -4995,7 +5005,9 @@ class Eynollah: contours_only_text_parent_d_ordered = [] contours_only_text_parent_d = [] #contours_only_text_parent = [] - + + boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) + if not len(contours_only_text_parent): # stop early empty_marginals = [[]] * len(polygons_of_marginals) @@ -5031,7 +5043,6 @@ class Eynollah: contours_only_text_parent, self.image, slope_first, confidence_matrix, map=self.executor.map) #print("text region early 4 in %.1fs", time.time() - t0) boxes_text, _ = get_text_region_boxes_by_given_contours(contours_only_text_parent) - boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) #print("text region early 5 in %.1fs", time.time() - t0) ## birdan sora chock chakir if not self.curved_line: From 733af1e9a71b31ab5902a9630ded787411255b76 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 17:43:32 +0200 Subject: [PATCH 302/374] :memo: update train/README.md, align with docs/train.md --- docs/train.md | 135 +++++++++++++++++++++++++++++------------------- train/README.md | 107 ++++++++++++++------------------------ 2 files changed, 120 insertions(+), 122 deletions(-) diff --git a/docs/train.md b/docs/train.md index b920a07..839529f 100644 --- a/docs/train.md +++ b/docs/train.md @@ -1,18 +1,24 @@ # Training documentation -This aims to assist users in preparing training datasets, training models, and +This document aims to assist users in preparing training datasets, training models, and performing inference with trained models. We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based reading order detection. For each use case, we provide guidance on how to generate the corresponding training dataset. The following three tasks can all be accomplished using the code in the -[`train`](https://github.com/qurator-spk/sbb_pixelwise_segmentation/tree/unifying-training-models) directory: +[`train`](https://github.com/qurator-spk/eynollah/tree/main/train) directory: * generate training dataset * train a model * inference with the trained model +## Training , evaluation and output + +The train and evaluation folders should contain subfolders of `images` and `labels`. + +The output folder should be an empty folder where the output model will be written to. + ## Generate training dataset The script `generate_gt_for_training.py` is used for generating training datasets. As the results of the following @@ -66,7 +72,7 @@ to the image area, with a default value of zero. To run the dataset generator, u python generate_gt_for_training.py machine-based-reading-order \ -dx "dir of GT xml files" \ -domi "dir where output images will be written" \ - -docl "dir where the labels will be written" \ +"" -docl "dir where the labels will be written" \ -ih "height" \ -iw "width" \ -min "min area ratio" @@ -312,60 +318,59 @@ The following parameter configuration can be applied to all segmentation use cas its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for classification and machine-based reading order, as you can see in their example config files. -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we -* offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first -* apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. -* task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this -* parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be -* set to ``false``. -* n_batch: Number of batches at each iteration. -* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it -* should set to 1. And for the case of layout detection just the unique number of classes should be given. -* n_epochs: Number of epochs. -* input_height: This indicates the height of model's input. -* input_width: This indicates the width of model's input. -* weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved -* in a folder named "pretrained_model" in the same directory of "train.py" script. -* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. -* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. -* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. -* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" parameter. -* degrading: If ``true``, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. -* brightening: If ``true``, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. -* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. -* rotation: If ``true``, 90 degree rotation will be applied on image. -* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. -* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. -* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. -* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. -* flip_index: Type of flips. -* blur_k: Type of blurrings. -* scales: Scales of scaling. -* brightness: The amount of brightenings. -* thetha: Rotation angles. -* degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the +* `backbone_type`: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we + offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first + apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. +* `task`: The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". +* `patches`: If you want to break input images into smaller patches (input size of the model) you need to set this +* parameter to `true`. In the case that the model should see the image once, like page extraction, patches should be + set to ``false``. +* `n_batch`: Number of batches at each iteration. +* `n_classes`: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it + should set to 1. And for the case of layout detection just the unique number of classes should be given. +* `n_epochs`: Number of epochs. +* `input_height`: This indicates the height of model's input. +* `input_width`: This indicates the width of model's input. +* `weight_decay`: Weight decay of l2 regularization of model layers. +* `pretraining`: Set to `true` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved + in a folder named "pretrained_model" in the same directory of "train.py" script. +* `augmentation`: If you want to apply any kind of augmentation this parameter should first set to `true`. +* `flip_aug`: If `true`, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. +* `blur_aug`: If `true`, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. +* `scaling`: If `true`, scaling will be applied on image. Scale of scaling is given with "scales" parameter. +* `degrading`: If `true`, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. +* `brightening`: If `true`, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. +* `rotation_not_90`: If `true`, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. +* `rotation`: If `true`, 90 degree rotation will be applied on image. +* `binarization`: If `true`,Otsu thresholding will be applied to augment the input data with binarized images. +* `scaling_bluring`: If `true`, combination of scaling and blurring will be applied on image. +* `scaling_binarization`: If `true`, combination of scaling and binarization will be applied on image. +* `scaling_flip`: If `true`, combination of scaling and flip will be applied on image. +* `flip_index`: Type of flips. +* `blur_k`: Type of blurrings. +* `scales`: Scales of scaling. +* `brightness`: The amount of brightenings. +* `thetha`: Rotation angles. +* `degrade_scales`: The amount of degradings. +* `continue_training`: If `true`, it means that you have already trained a model and you would like to continue the training. So it is needed to providethe dir of trained model with "dir_of_start_model" and index for naming themodels. For example if you have already trained for 3 epochs then your lastindex is 2 and if you want to continue - from model_1.h5, you can set -``index_start`` to 3 to start naming model with index 3. -* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train + from model_1.h5, you can set `index_start` to 3 to start naming model with index 3. +* `weighted_loss`: If `true`, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to `true`the parameter "is_loss_soft_dice" should be ``false`` +* `data_is_provided`: If you have already provided the input data you can set this to `true`. Be sure that the train and eval data are in"dir_output".Since when once we provide training data we resize and augmentthem and then wewrite them in sub-directories train and eval in "dir_output". -* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. -* index_start: Starting index for saved models in the case that "continue_training" is ``true``. -* dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. -* transformer_num_patches_xy: Number of patches for vision transformer in x and y direction respectively. -* transformer_patchsize_x: Patch size of vision transformer patches in x direction. -* transformer_patchsize_y: Patch size of vision transformer patches in y direction. -* transformer_projection_dim: Transformer projection dimension. Default value is 64. -* transformer_mlp_head_units: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. -* transformer_layers: transformer layers. Default value is 8. -* transformer_num_heads: Transformer number of heads. Default value is 4. -* transformer_cnn_first: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. +* `dir_train`: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. +* `index_start`: Starting index for saved models in the case that "continue_training" is `true`. +* `dir_of_start_model`: Directory containing pretrained model to continue training the model in the case that "continue_training" is `true`. +* `transformer_num_patches_xy`: Number of patches for vision transformer in x and y direction respectively. +* `transformer_patchsize_x`: Patch size of vision transformer patches in x direction. +* `transformer_patchsize_y`: Patch size of vision transformer patches in y direction. +* `transformer_projection_dim`: Transformer projection dimension. Default value is 64. +* `transformer_mlp_head_units`: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. +* `transformer_layers`: transformer layers. Default value is 8. +* `transformer_num_heads`: Transformer number of heads. Default value is 4. +* `transformer_cnn_first`: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. In the case of segmentation and enhancement the train and evaluation directory should be as following. @@ -394,6 +399,30 @@ command, similar to the process for classification and reading order: #### Binarization +### Ground truth format + +Lables for each pixel are identified by a number. So if you have a +binary case, ``n_classes`` should be set to ``2`` and labels should +be ``0`` and ``1`` for each class and pixel. + +In the case of multiclass, just set ``n_classes`` to the number of classes +you have and the try to produce the labels by pixels set from ``0 , 1 ,2 .., n_classes-1``. +The labels format should be png. +Our lables are 3 channel png images but only information of first channel is used. +If you have an image label with height and width of 10, for a binary case the first channel should look like this: + + Label: [ [1, 0, 0, 1, 1, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + ..., + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] ] + + This means that you have an image by `10*10*3` and `pixel[0,0]` belongs + to class `1` and `pixel[0,1]` belongs to class `0`. + + A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/~vahid.rezanezhad/binarization_training_data_sample/), which contains images and lables folders. + + An example config json file for binarization can be like this: ```yaml diff --git a/train/README.md b/train/README.md index 7c69a10..5f6d326 100644 --- a/train/README.md +++ b/train/README.md @@ -1,17 +1,39 @@ -# Pixelwise Segmentation -> Pixelwise segmentation for document images +# Training eynollah + +This README explains the technical details of how to set up and run training, for detailed information on parameterization, see [`docs/train.md`](../docs/train.md) ## Introduction -This repository contains the source code for training an encoder model for document image segmentation. + +This folder contains the source code for training an encoder model for document image segmentation. ## Installation -Either clone the repository via `git clone https://github.com/qurator-spk/sbb_pixelwise_segmentation.git` or download and unpack the [ZIP](https://github.com/qurator-spk/sbb_pixelwise_segmentation/archive/master.zip). + +Clone the repository and install eynollah along with the dependencies necessary for training: + +```sh +git clone https://github.com/qurator-spk/eynollah +cd eynollah +pip install '.[training]' +``` ### Pretrained encoder -Download our pretrained weights and add them to a ``pretrained_model`` folder: -https://qurator-data.de/sbb_pixelwise_segmentation/pretrained_encoder/ + +Download our pretrained weights and add them to a `train/pretrained_model` folder: + +```sh +cd train +wget -O pretrained_model.tar.gz https://zenodo.org/records/17243320/files/pretrained_model_v0_5_1.tar.gz?download=1 +tar xf pretrained_model.tar.gz +``` + +### Binarization training data + +A small sample of training data for binarization experiment can be found [on +zenodo](https://zenodo.org/records/17243320/files/training_data_sample_binarization_v0_5_1.tar.gz?download=1), +which contains `images` and `labels` folders. ### Helpful tools + * [`pagexml2img`](https://github.com/qurator-spk/page2img) > Tool to extract 2-D or 3-D RGB images from PAGE-XML data. In the former case, the output will be 1 2-D image array which each class has filled with a pixel value. In the case of a 3-D RGB image, each class will be defined with a RGB value and beside images, a text file of classes will also be produced. @@ -20,71 +42,18 @@ each class will be defined with a RGB value and beside images, a text file of cl * [`ocrd-segment-extract-pages`](https://github.com/OCR-D/ocrd_segment/blob/master/ocrd_segment/extract_pages.py) > Extract region classes and their colours in mask (pseg) images. Allows the color map as free dict parameter, and comes with a default that mimics PageViewer's coloring for quick debugging; it also warns when regions do overlap. -## Usage - -### Train -To train a model, run: ``python train.py with config_params.json`` - ### Train using Docker -#### Build the Docker image +Build the Docker image: - ```bash - docker build -t model-training . - ``` -#### Run Docker image - ```bash - docker run --gpus all -v /host/path/to/entry_point_dir:/entry_point_dir model-training - ``` +```bash +cd train +docker build -t model-training . +``` -### Ground truth format -Lables for each pixel are identified by a number. So if you have a -binary case, ``n_classes`` should be set to ``2`` and labels should -be ``0`` and ``1`` for each class and pixel. +Run Docker image -In the case of multiclass, just set ``n_classes`` to the number of classes -you have and the try to produce the labels by pixels set from ``0 , 1 ,2 .., n_classes-1``. -The labels format should be png. -Our lables are 3 channel png images but only information of first channel is used. -If you have an image label with height and width of 10, for a binary case the first channel should look like this: - - Label: [ [1, 0, 0, 1, 1, 0, 0, 1, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - ..., - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] ] - - This means that you have an image by `10*10*3` and `pixel[0,0]` belongs - to class `1` and `pixel[0,1]` belongs to class `0`. - - A small sample of training data for binarization experiment can be found here, [Training data sample](https://qurator-data.de/~vahid.rezanezhad/binarization_training_data_sample/), which contains images and lables folders. - -### Training , evaluation and output -The train and evaluation folders should contain subfolders of images and labels. -The output folder should be an empty folder where the output model will be written to. - -### Parameter configuration -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be set to ``false``. -* n_batch: Number of batches at each iteration. -* n_classes: Number of classes. In the case of binary classification this should be 2. -* n_epochs: Number of epochs. -* input_height: This indicates the height of model's input. -* input_width: This indicates the width of model's input. -* weight_decay: Weight decay of l2 regularization of model layers. -* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. -* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" in train.py file. -* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" in train.py file. -* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" in train.py file. -* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" in train.py file. -* rotation: If ``true``, 90 degree rotation will be applied on image. -* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. -* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. -* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. -* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set "index_start" to 3 to start naming model with index 3. -* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". -* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. - -#### Additional documentation -Please check the [wiki](https://github.com/qurator-spk/sbb_pixelwise_segmentation/wiki). +```bash +cd train +docker run --gpus all -v $PWD:/entry_point_dir model-training +``` From 48266b1ee0cd5aa7dc971336257307d7f681ddc1 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 18:01:21 +0200 Subject: [PATCH 303/374] make training dependencies optional-dependencies of eynollah i.e. `pip install "eynollah[training]"` will install the requirements for training --- pyproject.toml | 13 ++++++++----- requirements-ocr.txt | 2 ++ requirements-plotting.txt | 1 + requirements-training.txt | 1 + train/requirements.txt | 7 +------ 5 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 requirements-ocr.txt create mode 100644 requirements-plotting.txt create mode 120000 requirements-training.txt diff --git a/pyproject.toml b/pyproject.toml index 8a63543..ec3e5f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,11 @@ license.file = "LICENSE" requires-python = ">=3.8" keywords = ["document layout analysis", "image segmentation"] -dynamic = ["dependencies", "version"] +dynamic = [ + "dependencies", + "optional-dependencies", + "version" +] classifiers = [ "Development Status :: 4 - Beta", @@ -25,10 +29,6 @@ classifiers = [ "Topic :: Scientific/Engineering :: Image Processing", ] -[project.optional-dependencies] -OCR = ["torch <= 2.0.1", "transformers <= 4.30.2"] -plotting = ["matplotlib"] - [project.scripts] eynollah = "eynollah.cli:main" ocrd-eynollah-segment = "eynollah.ocrd_cli:main" @@ -41,6 +41,9 @@ Repository = "https://github.com/qurator-spk/eynollah.git" [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} optional-dependencies.test = {file = ["requirements-test.txt"]} +optional-dependencies.OCR = {file = ["requirements-ocr.txt"]} +optional-dependencies.plotting = {file = ["requirements-plotting.txt"]} +optional-dependencies.training = {file = ["requirements-training.txt"]} [tool.setuptools.packages.find] where = ["src"] diff --git a/requirements-ocr.txt b/requirements-ocr.txt new file mode 100644 index 0000000..9f31ebb --- /dev/null +++ b/requirements-ocr.txt @@ -0,0 +1,2 @@ +torch <= 2.0.1 +transformers <= 4.30.2 diff --git a/requirements-plotting.txt b/requirements-plotting.txt new file mode 100644 index 0000000..6ccafc3 --- /dev/null +++ b/requirements-plotting.txt @@ -0,0 +1 @@ +matplotlib diff --git a/requirements-training.txt b/requirements-training.txt new file mode 120000 index 0000000..e1bc9c3 --- /dev/null +++ b/requirements-training.txt @@ -0,0 +1 @@ +train/requirements.txt \ No newline at end of file diff --git a/train/requirements.txt b/train/requirements.txt index d8f9003..4df9c2f 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -1,11 +1,6 @@ -tensorflow == 2.12.1 +# tensorflow == 2.12.1 # TODO why not tensorflow < 2.13 as in eynollah/requirements.txt sacred -opencv-python-headless seaborn tqdm imutils -numpy scipy -scikit-learn -shapely -click From f0ef2b5db27b8f6e8abcce2aef261fbcb8575793 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 18:10:13 +0200 Subject: [PATCH 304/374] remove unused imports --- train/gt_gen_utils.py | 4 +--- train/inference.py | 6 ------ 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/train/gt_gen_utils.py b/train/gt_gen_utils.py index 38d48ca..2828d7b 100644 --- a/train/gt_gen_utils.py +++ b/train/gt_gen_utils.py @@ -1,5 +1,3 @@ -import click -import sys import os import numpy as np import warnings @@ -9,7 +7,7 @@ import cv2 from shapely import geometry from pathlib import Path import matplotlib.pyplot as plt -from PIL import Image, ImageDraw, ImageFont +from PIL import ImageFont KERNEL = np.ones((5, 5), np.uint8) diff --git a/train/inference.py b/train/inference.py index 595cfe7..0bff0ec 100644 --- a/train/inference.py +++ b/train/inference.py @@ -3,12 +3,9 @@ import os import numpy as np import warnings import cv2 -import seaborn as sns from tensorflow.keras.models import load_model import tensorflow as tf from tensorflow.keras import backend as K -from tensorflow.keras import layers -import tensorflow.keras.losses from tensorflow.keras.layers import * from models import * from gt_gen_utils import * @@ -16,7 +13,6 @@ import click import json from tensorflow.python.keras import backend as tensorflow_backend import xml.etree.ElementTree as ET -import matplotlib.pyplot as plt with warnings.catch_warnings(): @@ -55,11 +51,9 @@ class sbb_predict: seg=seg[:,:,0] seg_img=np.zeros((np.shape(seg)[0],np.shape(seg)[1],3)).astype(np.uint8) - colors=sns.color_palette("hls", self.n_classes) for c in ann_u: c=int(c) - segl=(seg==c) seg_img[:,:,0][seg==c]=c seg_img[:,:,1][seg==c]=c seg_img[:,:,2][seg==c]=c From 4f5cdf314004b6bb0a409aee7b3525391f8afcc7 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 18:12:45 +0200 Subject: [PATCH 305/374] move training scripts to src/eynollah/training --- pyproject.toml | 1 + {train => src/eynollah/training}/__init__.py | 0 .../training}/build_model_load_pretrained_weights_and_save.py | 0 {train => src/eynollah/training}/generate_gt_for_training.py | 0 {train => src/eynollah/training}/gt_gen_utils.py | 0 {train => src/eynollah/training}/inference.py | 0 {train => src/eynollah/training}/metrics.py | 0 {train => src/eynollah/training}/models.py | 0 {train => src/eynollah/training}/train.py | 0 {train => src/eynollah/training}/utils.py | 0 10 files changed, 1 insertion(+) rename {train => src/eynollah/training}/__init__.py (100%) rename {train => src/eynollah/training}/build_model_load_pretrained_weights_and_save.py (100%) rename {train => src/eynollah/training}/generate_gt_for_training.py (100%) rename {train => src/eynollah/training}/gt_gen_utils.py (100%) rename {train => src/eynollah/training}/inference.py (100%) rename {train => src/eynollah/training}/metrics.py (100%) rename {train => src/eynollah/training}/models.py (100%) rename {train => src/eynollah/training}/train.py (100%) rename {train => src/eynollah/training}/utils.py (100%) diff --git a/pyproject.toml b/pyproject.toml index ec3e5f8..8ca6cff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,7 @@ classifiers = [ eynollah = "eynollah.cli:main" ocrd-eynollah-segment = "eynollah.ocrd_cli:main" ocrd-sbb-binarize = "eynollah.ocrd_cli_binarization:main" +eynollah-training = "eynollah.training.cli:main" [project.urls] Homepage = "https://github.com/qurator-spk/eynollah" diff --git a/train/__init__.py b/src/eynollah/training/__init__.py similarity index 100% rename from train/__init__.py rename to src/eynollah/training/__init__.py diff --git a/train/build_model_load_pretrained_weights_and_save.py b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py similarity index 100% rename from train/build_model_load_pretrained_weights_and_save.py rename to src/eynollah/training/build_model_load_pretrained_weights_and_save.py diff --git a/train/generate_gt_for_training.py b/src/eynollah/training/generate_gt_for_training.py similarity index 100% rename from train/generate_gt_for_training.py rename to src/eynollah/training/generate_gt_for_training.py diff --git a/train/gt_gen_utils.py b/src/eynollah/training/gt_gen_utils.py similarity index 100% rename from train/gt_gen_utils.py rename to src/eynollah/training/gt_gen_utils.py diff --git a/train/inference.py b/src/eynollah/training/inference.py similarity index 100% rename from train/inference.py rename to src/eynollah/training/inference.py diff --git a/train/metrics.py b/src/eynollah/training/metrics.py similarity index 100% rename from train/metrics.py rename to src/eynollah/training/metrics.py diff --git a/train/models.py b/src/eynollah/training/models.py similarity index 100% rename from train/models.py rename to src/eynollah/training/models.py diff --git a/train/train.py b/src/eynollah/training/train.py similarity index 100% rename from train/train.py rename to src/eynollah/training/train.py diff --git a/train/utils.py b/src/eynollah/training/utils.py similarity index 100% rename from train/utils.py rename to src/eynollah/training/utils.py From 2baf42e878732330c0df54927c55a1ef9a9c8b03 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 18:15:54 +0200 Subject: [PATCH 306/374] organize imports, use relative imports --- src/eynollah/training/generate_gt_for_training.py | 3 ++- src/eynollah/training/gt_gen_utils.py | 1 - src/eynollah/training/inference.py | 10 ++++++---- src/eynollah/training/train.py | 10 ++++++---- src/eynollah/training/utils.py | 5 +++-- 5 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/eynollah/training/generate_gt_for_training.py b/src/eynollah/training/generate_gt_for_training.py index 388fced..d378c3e 100644 --- a/src/eynollah/training/generate_gt_for_training.py +++ b/src/eynollah/training/generate_gt_for_training.py @@ -1,10 +1,11 @@ import click import json -from gt_gen_utils import * from tqdm import tqdm from pathlib import Path from PIL import Image, ImageDraw, ImageFont +from .gt_gen_utils import * + @click.group() def main(): pass diff --git a/src/eynollah/training/gt_gen_utils.py b/src/eynollah/training/gt_gen_utils.py index 2828d7b..2e3428b 100644 --- a/src/eynollah/training/gt_gen_utils.py +++ b/src/eynollah/training/gt_gen_utils.py @@ -6,7 +6,6 @@ from tqdm import tqdm import cv2 from shapely import geometry from pathlib import Path -import matplotlib.pyplot as plt from PIL import ImageFont diff --git a/src/eynollah/training/inference.py b/src/eynollah/training/inference.py index 0bff0ec..24837a1 100644 --- a/src/eynollah/training/inference.py +++ b/src/eynollah/training/inference.py @@ -1,19 +1,21 @@ import sys import os -import numpy as np import warnings +import json + +import numpy as np import cv2 from tensorflow.keras.models import load_model import tensorflow as tf from tensorflow.keras import backend as K from tensorflow.keras.layers import * -from models import * -from gt_gen_utils import * import click -import json from tensorflow.python.keras import backend as tensorflow_backend import xml.etree.ElementTree as ET +from .models import * +from .gt_gen_utils import * + with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index add878a..3b99807 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -1,20 +1,22 @@ import os import sys +import json + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import tensorflow as tf from tensorflow.compat.v1.keras.backend import set_session import warnings from tensorflow.keras.optimizers import * from sacred import Experiment -from models import * -from utils import * -from metrics import * from tensorflow.keras.models import load_model from tqdm import tqdm -import json from sklearn.metrics import f1_score from tensorflow.keras.callbacks import Callback +from .models import * +from .utils import * +from .metrics import * + class SaveWeightsAfterSteps(Callback): def __init__(self, save_interval, save_path, _config): super(SaveWeightsAfterSteps, self).__init__() diff --git a/src/eynollah/training/utils.py b/src/eynollah/training/utils.py index ead4887..1278be5 100644 --- a/src/eynollah/training/utils.py +++ b/src/eynollah/training/utils.py @@ -1,13 +1,14 @@ import os +import math +import random + import cv2 import numpy as np import seaborn as sns from scipy.ndimage.interpolation import map_coordinates from scipy.ndimage.filters import gaussian_filter -import random from tqdm import tqdm import imutils -import math from tensorflow.keras.utils import to_categorical from PIL import Image, ImageEnhance From 690d47444caab7a8f2ba5443c0cb1701383c46e3 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 18:36:28 +0200 Subject: [PATCH 307/374] make relative wildcard imports explicit --- pyproject.toml | 1 - ..._model_load_pretrained_weights_and_save.py | 9 ++--- .../training/generate_gt_for_training.py | 20 ++++++++++- src/eynollah/training/inference.py | 11 +++++-- src/eynollah/training/train.py | 33 +++++++++++++++---- 5 files changed, 55 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8ca6cff..ec3e5f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ classifiers = [ eynollah = "eynollah.cli:main" ocrd-eynollah-segment = "eynollah.ocrd_cli:main" ocrd-sbb-binarize = "eynollah.ocrd_cli_binarization:main" -eynollah-training = "eynollah.training.cli:main" [project.urls] Homepage = "https://github.com/qurator-spk/eynollah" diff --git a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py index 125611e..ce3d955 100644 --- a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py +++ b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py @@ -1,12 +1,7 @@ -import os -import sys import tensorflow as tf -import warnings from tensorflow.keras.optimizers import * -from sacred import Experiment -from models import * -from utils import * -from metrics import * + +from .models import resnet50_unet def configuration(): diff --git a/src/eynollah/training/generate_gt_for_training.py b/src/eynollah/training/generate_gt_for_training.py index d378c3e..3fd93ae 100644 --- a/src/eynollah/training/generate_gt_for_training.py +++ b/src/eynollah/training/generate_gt_for_training.py @@ -1,10 +1,28 @@ import click import json +import os from tqdm import tqdm from pathlib import Path from PIL import Image, ImageDraw, ImageFont +import cv2 +import numpy as np -from .gt_gen_utils import * +from eynollah.training.gt_gen_utils import ( + filter_contours_area_of_image, + find_format_of_given_filename_in_dir, + find_new_features_of_contours, + fit_text_single_line, + get_content_of_dir, + get_images_of_ground_truth, + get_layout_contours_for_visualization, + get_textline_contours_and_ocr_text, + get_textline_contours_for_visualization, + overlay_layout_on_image, + read_xml, + resize_image, + visualize_image_from_contours, + visualize_image_from_contours_layout +) @click.group() def main(): diff --git a/src/eynollah/training/inference.py b/src/eynollah/training/inference.py index 24837a1..998c8fc 100644 --- a/src/eynollah/training/inference.py +++ b/src/eynollah/training/inference.py @@ -13,9 +13,14 @@ import click from tensorflow.python.keras import backend as tensorflow_backend import xml.etree.ElementTree as ET -from .models import * -from .gt_gen_utils import * - +from .gt_gen_utils import ( + filter_contours_area_of_image, + find_new_features_of_contours, + read_xml, + resize_image, + update_list_and_return_first_with_length_bigger_than_one +) +from .models import PatchEncoder, Patches with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 3b99807..527bca6 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -2,20 +2,39 @@ import os import sys import json +from eynollah.training.metrics import soft_dice_loss, weighted_categorical_crossentropy + +from .models import ( + PatchEncoder, + Patches, + machine_based_reading_order_model, + resnet50_classifier, + resnet50_unet, + vit_resnet50_unet, + vit_resnet50_unet_transformer_before_cnn +) +from .utils import ( + data_gen, + generate_arrays_from_folder_reading_order, + generate_data_from_folder_evaluation, + generate_data_from_folder_training, + get_one_hot, + provide_patches, + return_number_of_total_training_data +) + os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import tensorflow as tf from tensorflow.compat.v1.keras.backend import set_session -import warnings -from tensorflow.keras.optimizers import * +from tensorflow.keras.optimizers import SGD, Adam from sacred import Experiment from tensorflow.keras.models import load_model from tqdm import tqdm from sklearn.metrics import f1_score from tensorflow.keras.callbacks import Callback -from .models import * -from .utils import * -from .metrics import * +import numpy as np +import cv2 class SaveWeightsAfterSteps(Callback): def __init__(self, save_interval, save_path, _config): @@ -47,8 +66,8 @@ def configuration(): def get_dirs_or_files(input_data): + image_input, labels_input = os.path.join(input_data, 'images/'), os.path.join(input_data, 'labels/') if os.path.isdir(input_data): - image_input, labels_input = os.path.join(input_data, 'images/'), os.path.join(input_data, 'labels/') # Check if training dir exists assert os.path.isdir(image_input), "{} is not a directory".format(image_input) assert os.path.isdir(labels_input), "{} is not a directory".format(labels_input) @@ -425,7 +444,7 @@ def run(_config, n_classes, n_epochs, input_height, #f1score_tot = [0] indexer_start = 0 - opt = SGD(learning_rate=0.01, momentum=0.9) + # opt = SGD(learning_rate=0.01, momentum=0.9) opt_adam = tf.keras.optimizers.Adam(learning_rate=0.0001) model.compile(loss="binary_crossentropy", optimizer = opt_adam,metrics=['accuracy']) From 1c043c586a972c4088d204b179b37d64eb44a39f Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 18:52:11 +0200 Subject: [PATCH 308/374] eynollah-training: all training CLI into single click group --- pyproject.toml | 1 + ..._model_load_pretrained_weights_and_save.py | 6 ++--- src/eynollah/training/cli.py | 26 +++++++++++++++++++ .../training/generate_gt_for_training.py | 3 --- src/eynollah/training/inference.py | 11 +++----- src/eynollah/training/train.py | 11 +++++--- 6 files changed, 41 insertions(+), 17 deletions(-) create mode 100644 src/eynollah/training/cli.py diff --git a/pyproject.toml b/pyproject.toml index ec3e5f8..ec99c99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ classifiers = [ [project.scripts] eynollah = "eynollah.cli:main" +eynollah-training = "eynollah.training.cli:main" ocrd-eynollah-segment = "eynollah.ocrd_cli:main" ocrd-sbb-binarize = "eynollah.ocrd_cli_binarization:main" diff --git a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py index ce3d955..40fc1fe 100644 --- a/src/eynollah/training/build_model_load_pretrained_weights_and_save.py +++ b/src/eynollah/training/build_model_load_pretrained_weights_and_save.py @@ -1,5 +1,5 @@ +import click import tensorflow as tf -from tensorflow.keras.optimizers import * from .models import resnet50_unet @@ -8,8 +8,8 @@ def configuration(): gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) - -if __name__ == '__main__': +@click.command() +def build_model_load_pretrained_weights_and_save(): n_classes = 2 input_height = 224 input_width = 448 diff --git a/src/eynollah/training/cli.py b/src/eynollah/training/cli.py new file mode 100644 index 0000000..8ab754d --- /dev/null +++ b/src/eynollah/training/cli.py @@ -0,0 +1,26 @@ +import os +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + +import click +import sys + +from .build_model_load_pretrained_weights_and_save import build_model_load_pretrained_weights_and_save +from .generate_gt_for_training import main as generate_gt_cli +from .inference import main as inference_cli +from .train import ex + +@click.command(context_settings=dict( + ignore_unknown_options=True, +)) +@click.argument('SACRED_ARGS', nargs=-1, type=click.UNPROCESSED) +def train_cli(sacred_args): + ex.run_commandline([sys.argv[0]] + list(sacred_args)) + +@click.group('training') +def main(): + pass + +main.add_command(build_model_load_pretrained_weights_and_save) +main.add_command(generate_gt_cli, 'generate-gt') +main.add_command(inference_cli, 'inference') +main.add_command(train_cli, 'train') diff --git a/src/eynollah/training/generate_gt_for_training.py b/src/eynollah/training/generate_gt_for_training.py index 3fd93ae..693cab8 100644 --- a/src/eynollah/training/generate_gt_for_training.py +++ b/src/eynollah/training/generate_gt_for_training.py @@ -581,6 +581,3 @@ def visualize_ocr_text(xml_file, dir_xml, dir_out): # Draw the text draw.text((text_x, text_y), ocr_texts[index], fill="black", font=font) image_text.save(os.path.join(dir_out, f_name+'.png')) - -if __name__ == "__main__": - main() diff --git a/src/eynollah/training/inference.py b/src/eynollah/training/inference.py index 998c8fc..3fa8fd6 100644 --- a/src/eynollah/training/inference.py +++ b/src/eynollah/training/inference.py @@ -20,7 +20,10 @@ from .gt_gen_utils import ( resize_image, update_list_and_return_first_with_length_bigger_than_one ) -from .models import PatchEncoder, Patches +from .models import ( + PatchEncoder, + Patches +) with warnings.catch_warnings(): warnings.simplefilter("ignore") @@ -675,9 +678,3 @@ def main(image, dir_in, model, patches, save, save_layout, ground_truth, xml_fil x=sbb_predict(image, dir_in, model, task, config_params_model, patches, save, save_layout, ground_truth, xml_file, out, min_area) x.run() -if __name__=="__main__": - main() - - - - diff --git a/src/eynollah/training/train.py b/src/eynollah/training/train.py index 527bca6..97736e0 100644 --- a/src/eynollah/training/train.py +++ b/src/eynollah/training/train.py @@ -2,9 +2,13 @@ import os import sys import json -from eynollah.training.metrics import soft_dice_loss, weighted_categorical_crossentropy +import click -from .models import ( +from eynollah.training.metrics import ( + soft_dice_loss, + weighted_categorical_crossentropy +) +from eynollah.training.models import ( PatchEncoder, Patches, machine_based_reading_order_model, @@ -13,7 +17,7 @@ from .models import ( vit_resnet50_unet, vit_resnet50_unet_transformer_before_cnn ) -from .utils import ( +from eynollah.training.utils import ( data_gen, generate_arrays_from_folder_reading_order, generate_data_from_folder_evaluation, @@ -142,7 +146,6 @@ def config_params(): dir_rgb_backgrounds = None dir_rgb_foregrounds = None - @ex.automain def run(_config, n_classes, n_epochs, input_height, input_width, weight_decay, weighted_loss, From f60e0543ab293212c2d0e5791c0efa8658cc0ac4 Mon Sep 17 00:00:00 2001 From: kba Date: Wed, 1 Oct 2025 19:16:58 +0200 Subject: [PATCH 309/374] training: update docs --- docs/train.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/train.md b/docs/train.md index 839529f..252bead 100644 --- a/docs/train.md +++ b/docs/train.md @@ -13,7 +13,7 @@ The following three tasks can all be accomplished using the code in the * train a model * inference with the trained model -## Training , evaluation and output +## Training, evaluation and output The train and evaluation folders should contain subfolders of `images` and `labels`. @@ -22,11 +22,13 @@ The output folder should be an empty folder where the output model will be writt ## Generate training dataset The script `generate_gt_for_training.py` is used for generating training datasets. As the results of the following -command demonstrates, the dataset generator provides three different commands: +command demonstrates, the dataset generator provides several subcommands: -`python generate_gt_for_training.py --help` +```sh +eynollah-training generate-gt --help +``` -These three commands are: +The three most important subcommands are: * image-enhancement * machine-based-reading-order @@ -38,7 +40,7 @@ Generating a training dataset for image enhancement is quite straightforward. Al high-resolution images. The training dataset can then be generated using the following command: ```sh -python generate_gt_for_training.py image-enhancement \ +eynollah-training image-enhancement \ -dis "dir of high resolution images" \ -dois "dir where degraded images will be written" \ -dols "dir where the corresponding high resolution image will be written as label" \ @@ -69,7 +71,7 @@ to filter out regions smaller than this minimum size. This minimum size is defin to the image area, with a default value of zero. To run the dataset generator, use the following command: ```shell -python generate_gt_for_training.py machine-based-reading-order \ +eynollah-training generate-gt machine-based-reading-order \ -dx "dir of GT xml files" \ -domi "dir where output images will be written" \ "" -docl "dir where the labels will be written" \ @@ -144,7 +146,7 @@ region" are also present in the label. However, other regions like "noise region included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. ```sh -python generate_gt_for_training.py pagexml2label \ +eynollah-training generate-gt pagexml2label \ -dx "dir of GT xml files" \ -do "dir where output label png files will be written" \ -cfg "custom config json file" \ @@ -198,7 +200,7 @@ provided to ensure that they are cropped in sync with the labels. This ensures t required for training are obtained. The command should resemble the following: ```sh -python generate_gt_for_training.py pagexml2label \ +eynollah-training generate-gt pagexml2label \ -dx "dir of GT xml files" \ -do "dir where output label png files will be written" \ -cfg "custom config json file" \ @@ -261,7 +263,7 @@ And the "dir_eval" the same structure as train directory: The classification model can be trained using the following command line: ```sh -python train.py with config_classification.json +eynollah-training train with config_classification.json ``` As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. @@ -395,7 +397,9 @@ And the "dir_eval" the same structure as train directory: After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following command, similar to the process for classification and reading order: -`python train.py with config_classification.json` +``` +eynollah-training train with config_classification.json` +``` #### Binarization @@ -679,7 +683,7 @@ For conducting inference with a trained model, you simply need to execute the fo directory of the model and the image on which to perform inference: ```sh -python inference.py -m "model dir" -i "image" +eynollah-training inference -m "model dir" -i "image" ``` This will straightforwardly return the class of the image. @@ -691,7 +695,7 @@ without the reading order. We simply need to provide the model directory, the XM new XML file with the added reading order will be written to the output directory with the same name. We need to run: ```sh -python inference.py \ +eynollah-training inference \ -m "model dir" \ -xml "page xml file" \ -o "output dir to write new xml with reading order" @@ -702,7 +706,7 @@ python inference.py \ For conducting inference with a trained model for segmentation and enhancement you need to run the following command line: ```sh -python inference.py \ +eynollah-training inference \ -m "model dir" \ -i "image" \ -p \ From 8a9b4f8f55de9a2e51fd72af1894e771fe44f348 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 2 Oct 2025 12:16:26 +0200 Subject: [PATCH 310/374] remove commented-out requirement for tf == 2.12.1, rely on same version as in eynollah proper --- train/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/train/requirements.txt b/train/requirements.txt index 4df9c2f..2fb9908 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -1,4 +1,3 @@ -# tensorflow == 2.12.1 # TODO why not tensorflow < 2.13 as in eynollah/requirements.txt sacred seaborn tqdm From 0b9d4901a61ea777fc0db6e90930a734fe33302d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 20:51:03 +0200 Subject: [PATCH 311/374] contour features: avoid unused calculations, simplify, add shortcuts - new function: `find_center_of_contours` - simplified: `find_(new_)features_of_contours` --- src/eynollah/utils/contour.py | 78 ++++++++++++----------------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 0700ed4..041cbf6 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -79,61 +79,37 @@ def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area=1. found_polygons_early.append(polygon2contour(polygon)) return found_polygons_early -def find_new_features_of_contours(contours_main): - areas_main = np.array([cv2.contourArea(contours_main[j]) - for j in range(len(contours_main))]) - M_main = [cv2.moments(contours_main[j]) - for j in range(len(contours_main))] - cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) - for j in range(len(M_main))] - cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) - for j in range(len(M_main))] - try: - x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) - for j in range(len(contours_main))]) - argmin_x_main = np.array([np.argmin(contours_main[j][:, 0, 0]) - for j in range(len(contours_main))]) - x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 0] - for j in range(len(contours_main))]) - y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 1] - for j in range(len(contours_main))]) - x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) - for j in range(len(contours_main))]) - y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) - for j in range(len(contours_main))]) - y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) - for j in range(len(contours_main))]) - except: - x_min_main = np.array([np.min(contours_main[j][:, 0]) - for j in range(len(contours_main))]) - argmin_x_main = np.array([np.argmin(contours_main[j][:, 0]) - for j in range(len(contours_main))]) - x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0] - for j in range(len(contours_main))]) - y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 1] - for j in range(len(contours_main))]) - x_max_main = np.array([np.max(contours_main[j][:, 0]) - for j in range(len(contours_main))]) - y_min_main = np.array([np.min(contours_main[j][:, 1]) - for j in range(len(contours_main))]) - y_max_main = np.array([np.max(contours_main[j][:, 1]) - for j in range(len(contours_main))]) - # dis_x=np.abs(x_max_main-x_min_main) +def find_center_of_contours(contours): + moments = [cv2.moments(contour) for contour in contours] + cx = [feat["m10"] / (feat["m00"] + 1e-32) + for feat in moments] + cy = [feat["m01"] / (feat["m00"] + 1e-32) + for feat in moments] + return cx, cy - return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin +def find_new_features_of_contours(contours): + # areas = np.array([cv2.contourArea(contour) for contour in contours]) + cx, cy = find_center_of_contours(contours) + slice_x = np.index_exp[:, 0, 0] + slice_y = np.index_exp[:, 0, 1] + if any(contour.ndim < 3 for contour in contours): + slice_x = np.index_exp[:, 0] + slice_y = np.index_exp[:, 1] + x_min = np.array([np.min(contour[slice_x]) for contour in contours]) + x_max = np.array([np.max(contour[slice_x]) for contour in contours]) + y_min = np.array([np.min(contour[slice_y]) for contour in contours]) + y_max = np.array([np.max(contour[slice_y]) for contour in contours]) + # dis_x=np.abs(x_max-x_min) + y_corr_x_min = np.array([contour[np.argmin(contour[slice_x])][slice_y[1:]] + for contour in contours]) -def find_features_of_contours(contours_main): - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) + return cx, cy, x_min, x_max, y_min, y_max, y_corr_x_min - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) +def find_features_of_contours(contours): + y_min = np.array([np.min(contour[:,0,1]) for contour in contours]) + y_max = np.array([np.max(contour[:,0,1]) for contour in contours]) - return y_min_main, y_max_main + return y_min, y_max def return_parent_contours(contours, hierarchy): contours_parent = [contours[i] From 81827c2942e0a6b7e4121b9de510108de4f026fa Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 21:03:07 +0200 Subject: [PATCH 312/374] filter_contours_inside_a_bigger_one: simplify - use new `find_center_of_contours` - avoid loops in favour of array processing - use sets instead of `np.unique` and `np.delete` instead of list.pop --- src/eynollah/eynollah.py | 102 +++++++++++++++------------------------ 1 file changed, 39 insertions(+), 63 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 62ce002..b2d9016 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4208,7 +4208,7 @@ class Eynollah: return generated_text def return_list_of_contours_with_desired_order(self, ls_cons, sorted_indexes): - return [ls_cons[sorted_indexes[index]] for index in range(len(sorted_indexes))] + return list(np.array(ls_cons)[np.array(sorted_indexes)]) def return_it_in_two_groups(self, x_differential): split = [ind if x_differential[ind]!=x_differential[ind+1] else -1 @@ -4237,47 +4237,38 @@ class Eynollah: def filter_contours_inside_a_bigger_one(self, contours, contours_d_ordered, image, marginal_cnts=None, type_contour="textregion"): - if type_contour=="textregion": - areas = [cv2.contourArea(contours[j]) for j in range(len(contours))] + if type_contour == "textregion": + areas = np.array(list(map(cv2.contourArea, contours))) area_tot = image.shape[0]*image.shape[1] + areas_ratio = areas / area_tot + cx_main, cy_main = find_center_of_contours(contours) - M_main = [cv2.moments(contours[j]) - for j in range(len(contours))] - cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + contours_index_small = np.flatnonzero(areas_ratio < 1e-3) + contours_index_large = np.flatnonzero(areas_ratio >= 1e-3) - areas_ratio = np.array(areas)/ area_tot - contours_index_small = [ind for ind in range(len(contours)) if areas_ratio[ind] < 1e-3] - contours_index_big = [ind for ind in range(len(contours)) if areas_ratio[ind] >= 1e-3] - - #contours_> = [contours[ind] for ind in contours_index_big] + #contours_> = [contours[ind] for ind in contours_index_large] indexes_to_be_removed = [] for ind_small in contours_index_small: - results = [cv2.pointPolygonTest(contours[ind], (cx_main[ind_small], cy_main[ind_small]), False) - for ind in contours_index_big] - if marginal_cnts: - results_marginal = [cv2.pointPolygonTest(marginal_cnts[ind], + results = [cv2.pointPolygonTest(contours[ind_large], (cx_main[ind_small], + cy_main[ind_small]), + False) + for ind_large in contours_index_large] + results = np.array(results) + if np.any(results==1): + indexes_to_be_removed.append(ind_small) + elif marginal_cnts: + results_marginal = [cv2.pointPolygonTest(marginal_cnt, (cx_main[ind_small], cy_main[ind_small]), False) - for ind in range(len(marginal_cnts))] + for marginal_cnt in marginal_cnts] results_marginal = np.array(results_marginal) - if np.any(results_marginal==1): indexes_to_be_removed.append(ind_small) - results = np.array(results) - - if np.any(results==1): - indexes_to_be_removed.append(ind_small) - - if len(indexes_to_be_removed)>0: - indexes_to_be_removed = np.unique(indexes_to_be_removed) - indexes_to_be_removed = np.sort(indexes_to_be_removed)[::-1] - for ind in indexes_to_be_removed: - contours.pop(ind) - if len(contours_d_ordered)>0: - contours_d_ordered.pop(ind) + contours = np.delete(contours, indexes_to_be_removed, axis=0) + if len(contours_d_ordered): + contours_d_ordered = np.delete(contours_d_ordered, indexes_to_be_removed, axis=0) return contours, contours_d_ordered @@ -4285,33 +4276,21 @@ class Eynollah: contours_txtline_of_all_textregions = [] indexes_of_textline_tot = [] index_textline_inside_textregion = [] + for ind_region, textlines in enumerate(contours): + contours_txtline_of_all_textregions.extend(textlines) + index_textline_inside_textregion.extend(list(range(len(textlines)))) + indexes_of_textline_tot.extend([ind_region] * len(textlines)) - for jj in range(len(contours)): - contours_txtline_of_all_textregions = contours_txtline_of_all_textregions + contours[jj] - - ind_textline_inside_tr = list(range(len(contours[jj]))) - index_textline_inside_textregion = index_textline_inside_textregion + ind_textline_inside_tr - ind_ins = [jj] * len(contours[jj]) - indexes_of_textline_tot = indexes_of_textline_tot + ind_ins - - M_main_tot = [cv2.moments(contours_txtline_of_all_textregions[j]) - for j in range(len(contours_txtline_of_all_textregions))] - cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - - areas_tot = [cv2.contourArea(con_ind) for con_ind in contours_txtline_of_all_textregions] + areas_tot = np.array(list(map(cv2.contourArea, contours_txtline_of_all_textregions))) area_tot_tot = image.shape[0]*image.shape[1] + cx_main_tot, cy_main_tot = find_center_of_contours(contours_txtline_of_all_textregions) - textregion_index_to_del = [] - textline_in_textregion_index_to_del = [] + textline_in_textregion_index_to_del = {} for ij in range(len(contours_txtline_of_all_textregions)): - args_all = list(np.array(range(len(contours_txtline_of_all_textregions)))) - args_all.pop(ij) - - areas_without = np.array(areas_tot)[args_all] area_of_con_interest = areas_tot[ij] - - args_with_bigger_area = np.array(args_all)[areas_without > 1.5*area_of_con_interest] + args_without = np.delete(np.arange(len(contours_txtline_of_all_textregions)), ij) + areas_without = areas_tot[args_without] + args_with_bigger_area = args_without[areas_without > 1.5*area_of_con_interest] if len(args_with_bigger_area)>0: results = [cv2.pointPolygonTest(contours_txtline_of_all_textregions[ind], @@ -4322,18 +4301,15 @@ class Eynollah: results = np.array(results) if np.any(results==1): #print(indexes_of_textline_tot[ij], index_textline_inside_textregion[ij]) - textregion_index_to_del.append(int(indexes_of_textline_tot[ij])) - textline_in_textregion_index_to_del.append(int(index_textline_inside_textregion[ij])) - #contours[int(indexes_of_textline_tot[ij])].pop(int(index_textline_inside_textregion[ij])) + textline_in_textregion_index_to_del.setdefault( + indexes_of_textline_tot[ij], list()).append( + index_textline_inside_textregion[ij]) + #contours[indexes_of_textline_tot[ij]].pop(index_textline_inside_textregion[ij]) - textregion_index_to_del = np.array(textregion_index_to_del) - textline_in_textregion_index_to_del = np.array(textline_in_textregion_index_to_del) - for ind_u_a_trs in np.unique(textregion_index_to_del): - textline_in_textregion_index_to_del_ind = \ - textline_in_textregion_index_to_del[textregion_index_to_del==ind_u_a_trs] - textline_in_textregion_index_to_del_ind = np.sort(textline_in_textregion_index_to_del_ind)[::-1] - for ittrd in textline_in_textregion_index_to_del_ind: - contours[ind_u_a_trs].pop(ittrd) + for textregion_index_to_del in textline_in_textregion_index_to_del: + contours[textregion_index_to_del] = list(np.delete( + contours[textregion_index_to_del], + textline_in_textregion_index_to_del[textregion_index_to_del])) return contours From 8869c20c33c673e02e4f60081b96a8bd71d823d2 Mon Sep 17 00:00:00 2001 From: vahidrezanezhad Date: Mon, 6 Oct 2025 14:53:47 +0200 Subject: [PATCH 313/374] updating CHANGELOG for v0.5.0 --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfdd1ce..70e8854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,12 @@ Versioned according to [Semantic Versioning](http://semver.org/). Fixed: * restoring the contour in the original image caused an error due to an empty tuple, #154 + * removed NumPy warnings (fixed issue #158) + * fixed issue #124 + * Drop capitals are now handled separately from their corresponding textline + * Marginals are now divided into left and right. Their reading order is written first for left marginals, then for right marginals, and within each side from top to bottom + * Added a new page extraction model. Instead of bounding boxes, it outputs page contours in the XML file, improving results for skewed pages + * Improved reading order for cases where a textline is segmented into multiple smaller textlines Changed @@ -24,6 +30,20 @@ Added: * `eynollah machine-based-reading-order` CLI to run reading order detection, #175 * `eynollah enhancement` CLI to run image enhancement, #175 * Improved models for page extraction and reading order detection, #175 + * For the lightweight version (layout and textline detection), thresholds are now assigned to the artificial class. Users can apply these thresholds to improve detection of isolated textlines and regions. To counteract the drawback of thresholding, the skeleton of the artificial class is used to keep lines as thin as possible (resolved issues #163 and #161) + * Added and integrated trained CNN-RNN OCR models + * Added and integrated a trained TrOCR model + * Improved OCR detection to support vertical and curved textlines + * Introduced a new machine-based reading order model with rotation augmentation + * Optimized reading order speed by clustering text regions that belong to the same block, maintaining top-to-bottom order + * Implemented text merging across textlines based on hyphenation when a line ends with a hyphen + * Integrated image enhancement as a separate use case + * Added reading order functionality on the layout level as a separate use case + * CNN-RNN OCR models provide confidence scores for predictions + * Added OCR visualization: predicted OCR can be overlaid on an image of the same size as the input + * Introduced a threshold value for CNN-RNN OCR models, allowing users to filter out low-confidence textline predictions + * For OCR, users can specify a single model by name instead of always using the default model + * Under the OCR use case, if ground-truth XMLs and images are available, textline image and corresponding text extraction can now be performed Merged PRs: From 4ffe6190d2c6b885b27330027f4a0d8fd97a32f6 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 9 Oct 2025 14:03:26 +0200 Subject: [PATCH 314/374] :memo: changelog --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70e8854..5ca95a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). Fixed: * restoring the contour in the original image caused an error due to an empty tuple, #154 - * removed NumPy warnings (fixed issue #158) - * fixed issue #124 + * removed NumPy warnings calculating sigma, mean, (fixed issue #158) + * fixed bug in `separate_lines.py`, #124 * Drop capitals are now handled separately from their corresponding textline * Marginals are now divided into left and right. Their reading order is written first for left marginals, then for right marginals, and within each side from top to bottom * Added a new page extraction model. Instead of bounding boxes, it outputs page contours in the XML file, improving results for skewed pages @@ -31,7 +31,7 @@ Added: * `eynollah enhancement` CLI to run image enhancement, #175 * Improved models for page extraction and reading order detection, #175 * For the lightweight version (layout and textline detection), thresholds are now assigned to the artificial class. Users can apply these thresholds to improve detection of isolated textlines and regions. To counteract the drawback of thresholding, the skeleton of the artificial class is used to keep lines as thin as possible (resolved issues #163 and #161) - * Added and integrated trained CNN-RNN OCR models + * Added and integrated a trained CNN-RNN OCR models * Added and integrated a trained TrOCR model * Improved OCR detection to support vertical and curved textlines * Introduced a new machine-based reading order model with rotation augmentation @@ -43,7 +43,7 @@ Added: * Added OCR visualization: predicted OCR can be overlaid on an image of the same size as the input * Introduced a threshold value for CNN-RNN OCR models, allowing users to filter out low-confidence textline predictions * For OCR, users can specify a single model by name instead of always using the default model - * Under the OCR use case, if ground-truth XMLs and images are available, textline image and corresponding text extraction can now be performed + * Under the OCR use case, if Ground Truth XMLs and images are available, textline image and corresponding text extraction can now be performed Merged PRs: From 8c3d5eb0eb0eccd97542a86b0d3385e95f4f1da0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 21:07:35 +0200 Subject: [PATCH 315/374] separate_marginals_to_left_and_right_and_order_from_top_to_down: simplify - use new `find_center_of_contours` - avoid loops in favour of array processing - avoid repeated sorting --- src/eynollah/eynollah.py | 75 +++++++++++++++++----------------- src/eynollah/utils/__init__.py | 2 +- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index b2d9016..9eba3d3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4418,52 +4418,53 @@ class Eynollah: def separate_marginals_to_left_and_right_and_order_from_top_to_down( self, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes_marginals, mid_point_of_page_width): - cx_marg, cy_marg, _, _, _, _, _ = find_new_features_of_contours( - polygons_of_marginals) - + cx_marg, cy_marg = find_center_of_contours(polygons_of_marginals) cx_marg = np.array(cx_marg) cy_marg = np.array(cy_marg) + + def split(lis): + array = np.array(lis) + return (list(array[cx_marg < mid_point_of_page_width]), + list(array[cx_marg >= mid_point_of_page_width])) + + (poly_marg_left, + poly_marg_right) = \ + split(polygons_of_marginals) + + (all_found_textline_polygons_marginals_left, + all_found_textline_polygons_marginals_right) = \ + split(all_found_textline_polygons_marginals) - poly_marg_left = list( np.array(polygons_of_marginals)[cx_marg < mid_point_of_page_width] ) - poly_marg_right = list( np.array(polygons_of_marginals)[cx_marg >= mid_point_of_page_width] ) + (all_box_coord_marginals_left, + all_box_coord_marginals_right) = \ + split(all_box_coord_marginals) - all_found_textline_polygons_marginals_left = \ - list( np.array(all_found_textline_polygons_marginals)[cx_marg < mid_point_of_page_width] ) - all_found_textline_polygons_marginals_right = \ - list( np.array(all_found_textline_polygons_marginals)[cx_marg >= mid_point_of_page_width] ) + (slopes_marg_left, + slopes_marg_right) = \ + split(slopes_marginals) - all_box_coord_marginals_left = list( np.array(all_box_coord_marginals)[cx_marg < mid_point_of_page_width] ) - all_box_coord_marginals_right = list( np.array(all_box_coord_marginals)[cx_marg >= mid_point_of_page_width] ) + (cy_marg_left, + cy_marg_right) = \ + split(cy_marg) + + order_left = np.argsort(cy_marg_left) + order_right = np.argsort(cy_marg_right) + def sort_left(lis): + return list(np.array(lis)[order_left]) + def sort_right(lis): + return list(np.array(lis)[order_right]) - slopes_marg_left = list( np.array(slopes_marginals)[cx_marg < mid_point_of_page_width] ) - slopes_marg_right = list( np.array(slopes_marginals)[cx_marg >= mid_point_of_page_width] ) + ordered_left_marginals = sort_left(poly_marg_left) + ordered_right_marginals = sort_right(poly_marg_right) - cy_marg_left = cy_marg[cx_marg < mid_point_of_page_width] - cy_marg_right = cy_marg[cx_marg >= mid_point_of_page_width] + ordered_left_marginals_textline = sort_left(all_found_textline_polygons_marginals_left) + ordered_right_marginals_textline = sort_right(all_found_textline_polygons_marginals_right) - ordered_left_marginals = [poly for _, poly in sorted(zip(cy_marg_left, poly_marg_left), - key=lambda x: x[0])] - ordered_right_marginals = [poly for _, poly in sorted(zip(cy_marg_right, poly_marg_right), - key=lambda x: x[0])] + ordered_left_marginals_bbox = sort_left(all_box_coord_marginals_left) + ordered_right_marginals_bbox = sort_right(all_box_coord_marginals_right) - ordered_left_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_left, - all_found_textline_polygons_marginals_left), - key=lambda x: x[0])] - ordered_right_marginals_textline = [poly for _, poly in sorted(zip(cy_marg_right, - all_found_textline_polygons_marginals_right), - key=lambda x: x[0])] - - ordered_left_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_left, - all_box_coord_marginals_left), - key=lambda x: x[0])] - ordered_right_marginals_bbox = [poly for _, poly in sorted(zip(cy_marg_right, - all_box_coord_marginals_right), - key=lambda x: x[0])] - - ordered_left_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_left, slopes_marg_left), - key=lambda x: x[0])] - ordered_right_slopes_marginals = [poly for _, poly in sorted(zip(cy_marg_right, slopes_marg_right), - key=lambda x: x[0])] + ordered_left_slopes_marginals = sort_left(slopes_marg_left) + ordered_right_slopes_marginals = sort_right(slopes_marg_right) return (ordered_left_marginals, ordered_right_marginals, diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 52bf3ef..4eee5a9 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1417,7 +1417,7 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) ret, thresh = cv2.threshold(imgray, 0, 255, 0) contours_cross,_=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - cx_cross,cy_cross ,_ , _, _ ,_,_=find_new_features_of_contours(contours_cross) + cx_cross, cy_cross = find_center_of_contours(contours_cross) for ii in range(len(cx_cross)): img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])+5:int(cx_cross[ii])+40,0]=0 img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])-40:int(cx_cross[ii])-4,0]=0 From 3f3353ec3a53384a100ef9ebe2fefb7e092e968c Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 21:28:04 +0200 Subject: [PATCH 316/374] do_order_of_regions: simplify - avoid loops in favour of array processing --- src/eynollah/eynollah.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 9eba3d3..7f7f53f 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2518,6 +2518,8 @@ class Eynollah: self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): self.logger.debug("enter do_order_of_regions_full_layout") + contours_only_text_parent = np.array(contours_only_text_parent) + contours_only_text_parent_h = np.array(contours_only_text_parent_h) boxes = np.array(boxes, dtype=int) # to be on the safe side cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours( contours_only_text_parent) @@ -2573,14 +2575,9 @@ class Eynollah: xs = slice(*boxes[iij][0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] - con_inter_box = [] - con_inter_box_h = [] + con_inter_box = contours_only_text_parent[args_contours_box] + con_inter_box_h = contours_only_text_parent_h[args_contours_box_h] - for box in args_contours_box: - con_inter_box.append(contours_only_text_parent[box]) - - for box in args_contours_box_h: - con_inter_box_h.append(contours_only_text_parent_h[box]) indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) @@ -2675,14 +2672,8 @@ class Eynollah: xs = slice(*boxes[iij][0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] - con_inter_box = [] - con_inter_box_h = [] - - for box in args_contours_box: - con_inter_box.append(contours_only_text_parent[box]) - - for box in args_contours_box_h: - con_inter_box_h.append(contours_only_text_parent_h[box]) + con_inter_box = contours_only_text_parent[args_contours_box] + con_inter_box_h = contours_only_text_parent_h[args_contours_box_h] indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) @@ -2729,6 +2720,8 @@ class Eynollah: self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): self.logger.debug("enter do_order_of_regions_no_full_layout") + contours_only_text_parent = np.array(contours_only_text_parent) + contours_only_text_parent_h = np.array(contours_only_text_parent_h) boxes = np.array(boxes, dtype=int) # to be on the safe side cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours( contours_only_text_parent) @@ -2761,10 +2754,8 @@ class Eynollah: ys = slice(*boxes[iij][2:4]) xs = slice(*boxes[iij][0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] - con_inter_box = [] + con_inter_box = contours_only_text_parent[args_contours_box] con_inter_box_h = [] - for i in range(len(args_contours_box)): - con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) From 415b2cbad843d4fa083f94f459777af97bd81234 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 21:36:22 +0200 Subject: [PATCH 317/374] eynollah, drop_capitals: simplify - use new `find_center_of_contours` --- src/eynollah/eynollah.py | 21 ++++++++------------- src/eynollah/utils/drop_capitals.py | 27 ++++++++++++++------------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 7f7f53f..357c0c2 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -70,6 +70,7 @@ from .utils.contour import ( filter_contours_area_of_image, filter_contours_area_of_image_tables, find_contours_mean_y_diff, + find_center_of_contours, find_new_features_of_contours, find_features_of_contours, get_text_region_boxes_by_given_contours, @@ -1859,14 +1860,10 @@ class Eynollah: def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): polygons_of_textlines = return_contours_of_interested_region(textline_mask_tot,1,0.00001) - M_main_tot = [cv2.moments(polygons_of_textlines[j]) - for j in range(len(polygons_of_textlines))] + cx_main_tot, cy_main_tot = find_center_of_contours(polygons_of_textlines) + w_h_textlines = [cv2.boundingRect(polygon)[2:] for polygon in polygons_of_textlines] - w_h_textlines = [cv2.boundingRect(polygons_of_textlines[i])[2:] for i in range(len(polygons_of_textlines))] - cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - - args_textlines = np.array(range(len(polygons_of_textlines))) + args_textlines = np.arange(len(polygons_of_textlines)) all_found_textline_polygons = [] slopes = [] all_box_coord =[] @@ -4809,8 +4806,8 @@ class Eynollah: areas_cnt_text_parent = self.return_list_of_contours_with_desired_order( areas_cnt_text_parent, index_con_parents) - cx_bigest_big, cy_biggest_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest]) - cx_bigest, cy_biggest, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent) + cx_bigest_big, cy_biggest_big = find_center_of_contours([contours_biggest]) + cx_bigest, cy_biggest = find_center_of_contours(contours_only_text_parent) if np.abs(slope_deskew) >= SLOPE_THRESHOLD: contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) @@ -4834,10 +4831,8 @@ class Eynollah: areas_cnt_text_d = self.return_list_of_contours_with_desired_order( areas_cnt_text_d, index_con_parents_d) - cx_bigest_d_big, cy_biggest_d_big, _, _, _, _, _ = \ - find_new_features_of_contours([contours_biggest_d]) - cx_bigest_d, cy_biggest_d, _, _, _, _, _ = \ - find_new_features_of_contours(contours_only_text_parent_d) + cx_bigest_d_big, cy_biggest_d_big = find_center_of_contours([contours_biggest_d]) + cx_bigest_d, cy_biggest_d = find_center_of_contours(contours_only_text_parent_d) try: if len(cx_bigest_d) >= 5: cx_bigest_d_last5 = cx_bigest_d[-5:] diff --git a/src/eynollah/utils/drop_capitals.py b/src/eynollah/utils/drop_capitals.py index 67547d3..9f82fac 100644 --- a/src/eynollah/utils/drop_capitals.py +++ b/src/eynollah/utils/drop_capitals.py @@ -1,6 +1,7 @@ import numpy as np import cv2 from .contour import ( + find_center_of_contours, find_new_features_of_contours, return_contours_of_image, return_parent_contours, @@ -22,8 +23,8 @@ def adhere_drop_capital_region_into_corresponding_textline( ): # print(np.shape(all_found_textline_polygons),np.shape(all_found_textline_polygons[3]),'all_found_textline_polygonsshape') # print(all_found_textline_polygons[3]) - cx_m, cy_m, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent) - cx_h, cy_h, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent_h) + cx_m, cy_m = find_center_of_contours(contours_only_text_parent) + cx_h, cy_h = find_center_of_contours(contours_only_text_parent_h) cx_d, cy_d, _, _, y_min_d, y_max_d, _ = find_new_features_of_contours(polygons_of_drop_capitals) img_con_all = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) @@ -89,9 +90,9 @@ def adhere_drop_capital_region_into_corresponding_textline( region_final = region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)] - 1 # print(region_final,'region_final') - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) try: - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -153,9 +154,9 @@ def adhere_drop_capital_region_into_corresponding_textline( # areas_main=np.array([cv2.contourArea(all_found_textline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_textline_polygons[int(region_final)]))]) - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) try: - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -208,7 +209,7 @@ def adhere_drop_capital_region_into_corresponding_textline( try: # print(all_found_textline_polygons[j_cont][0]) - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -261,7 +262,7 @@ def adhere_drop_capital_region_into_corresponding_textline( else: pass - ##cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + ##cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) ###print(all_box_coord[j_cont]) ###print(cx_t) ###print(cy_t) @@ -315,9 +316,9 @@ def adhere_drop_capital_region_into_corresponding_textline( region_final = region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)] - 1 # print(region_final,'region_final') - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) try: - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -375,12 +376,12 @@ def adhere_drop_capital_region_into_corresponding_textline( # areas_main=np.array([cv2.contourArea(all_found_textline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_textline_polygons[int(region_final)]))]) - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) # print(cx_t,'print') try: # print(all_found_textline_polygons[j_cont][0]) - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -453,7 +454,7 @@ def adhere_drop_capital_region_into_corresponding_textline( #####try: #####if len(contours_new_parent)==1: ######print(all_found_textline_polygons[j_cont][0]) - #####cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[j_cont]) + #####cx_t, cy_t = find_center_of_contours(all_found_textline_polygons[j_cont]) ######print(all_box_coord[j_cont]) ######print(cx_t) ######print(cy_t) From a1c8fd44677fc894395652de070710a5fc6aae2e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 21:41:37 +0200 Subject: [PATCH 318/374] do_order_of_regions / order_of_regions: simplify - array-convert only once (before returning from `order_of_regions`) - avoid passing `matrix_of_orders` unnecessarily between `order_of_regions` and `order_and_id_of_texts` --- src/eynollah/eynollah.py | 73 +++++++++++++++++----------------- src/eynollah/utils/__init__.py | 2 +- src/eynollah/utils/xml.py | 6 +-- 3 files changed, 38 insertions(+), 43 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 357c0c2..8351ab6 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2567,26 +2567,25 @@ class Eynollah: ref_point = 0 order_of_texts_tot = [] id_of_texts_tot = [] - for iij in range(len(boxes)): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) + for iij, box in enumerate(boxes): + ys = slice(*box[2:4]) + xs = slice(*box[0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] con_inter_box = contours_only_text_parent[args_contours_box] con_inter_box_h = contours_only_text_parent_h[args_contours_box_h] - - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) + indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( + textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) order_of_texts, id_of_texts = order_and_id_of_texts( con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] - indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] + indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] + indexes_sorted_head = indexes_sorted[kind_of_texts_sorted == 2] + indexes_by_type_head = index_by_kind_sorted[kind_of_texts_sorted == 2] for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] @@ -2664,25 +2663,25 @@ class Eynollah: ref_point = 0 order_of_texts_tot = [] id_of_texts_tot = [] - for iij, _ in enumerate(boxes): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) + for iij, box in enumerate(boxes): + ys = slice(*box[2:4]) + xs = slice(*box[0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] con_inter_box = contours_only_text_parent[args_contours_box] con_inter_box_h = contours_only_text_parent_h[args_contours_box_h] - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) + indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( + textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) order_of_texts, id_of_texts = order_and_id_of_texts( con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] - indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] + indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] + indexes_sorted_head = indexes_sorted[kind_of_texts_sorted == 2] + indexes_by_type_head = index_by_kind_sorted[kind_of_texts_sorted == 2] for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] @@ -2747,22 +2746,22 @@ class Eynollah: ref_point = 0 order_of_texts_tot = [] id_of_texts_tot = [] - for iij in range(len(boxes)): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) + for iij, box in enumerate(boxes): + ys = slice(*box[2:4]) + xs = slice(*box[0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] con_inter_box = contours_only_text_parent[args_contours_box] con_inter_box_h = [] - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) + indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( + textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) order_of_texts, id_of_texts = order_and_id_of_texts( con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] + indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] @@ -2808,24 +2807,24 @@ class Eynollah: ref_point = 0 order_of_texts_tot = [] id_of_texts_tot = [] - for iij in range(len(boxes)): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) + for iij, box in enumerate(boxes): + ys = slice(*box[2:4]) + xs = slice(*box[0:2]) args_contours_box = args_contours[np.array(arg_text_con) == iij] con_inter_box = [] con_inter_box_h = [] for i in range(len(args_contours_box)): con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) + indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( + textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) order_of_texts, id_of_texts = order_and_id_of_texts( con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] + indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 4eee5a9..27a85da 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1325,7 +1325,7 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): final_types.append(1) final_index_type.append(ind_missed) - return final_indexers_sorted, matrix_of_orders, final_types, final_index_type + return np.array(final_indexers_sorted), np.array(final_types), np.array(final_index_type) def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( img_p_in_ver, img_in_hor,num_col_classifier): diff --git a/src/eynollah/utils/xml.py b/src/eynollah/utils/xml.py index 13420df..a61dadb 100644 --- a/src/eynollah/utils/xml.py +++ b/src/eynollah/utils/xml.py @@ -65,11 +65,7 @@ def xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_margina og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=id_marginal)) region_counter.inc('region') -def order_and_id_of_texts(found_polygons_text_region, found_polygons_text_region_h, matrix_of_orders, indexes_sorted, index_of_types, kind_of_texts, ref_point): - indexes_sorted = np.array(indexes_sorted) - index_of_types = np.array(index_of_types) - kind_of_texts = np.array(kind_of_texts) - +def order_and_id_of_texts(found_polygons_text_region, found_polygons_text_region_h, indexes_sorted, index_of_types, kind_of_texts, ref_point): id_of_texts = [] order_of_texts = [] From 4950e6bd784e2078ca7b65b1fcbf20de29d0f613 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 22:28:52 +0200 Subject: [PATCH 319/374] order_of_regions: simplify - use new `find_center_of_contours` - avoid unused calculations - avoid loops in favour of array processing --- src/eynollah/utils/__init__.py | 131 +++++++++------------------------ 1 file changed, 34 insertions(+), 97 deletions(-) diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 27a85da..92da14a 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -15,10 +15,21 @@ from scipy.ndimage import gaussian_filter1d from .is_nan import isNaN from .contour import (contours_in_same_horizon, + find_center_of_contours, find_new_features_of_contours, return_contours_of_image, return_parent_contours) +def pairwise(iterable): + # pairwise('ABCDEFG') → AB BC CD DE EF FG + + iterator = iter(iterable) + a = next(iterator, None) + + for b in iterator: + yield a, b + a = b + def return_x_start_end_mothers_childs_and_type_of_reading_order( x_min_hor_some, x_max_hor_some, cy_hor_some, peak_points, cy_hor_diff): @@ -1183,106 +1194,45 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) textlines_con_changed.append(textlines_big_org_form) return textlines_con_changed -def order_of_regions(textline_mask, contours_main, contours_header, y_ref): +def order_of_regions(textline_mask, contours_main, contours_head, y_ref): ##plt.imshow(textline_mask) ##plt.show() - """ - print(len(contours_main),'contours_main') - mada_n=textline_mask.sum(axis=1) - y=mada_n[:] - - y_help=np.zeros(len(y)+40) - y_help[20:len(y)+20]=y - x=np.arange(len(y)) - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - ##plt.imshow(textline_mask[:,:]) - ##plt.show() - - sigma_gaus=8 - z= gaussian_filter1d(y_help, sigma_gaus) - zneg_rev=-y_help+np.max(y_help) - zneg=np.zeros(len(zneg_rev)+40) - zneg[20:len(zneg_rev)+20]=zneg_rev - zneg= gaussian_filter1d(zneg, sigma_gaus) - - peaks, _ = find_peaks(z, height=0) - peaks_neg, _ = find_peaks(zneg, height=0) - peaks_neg=peaks_neg-20-20 - peaks=peaks-20 - """ - textline_sum_along_width = textline_mask.sum(axis=1) - - y = textline_sum_along_width[:] + y = textline_mask.sum(axis=1) # horizontal projection profile y_padded = np.zeros(len(y) + 40) y_padded[20 : len(y) + 20] = y - x = np.arange(len(y)) - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) sigma_gaus = 8 - z = gaussian_filter1d(y_padded, sigma_gaus) - zneg_rev = -y_padded + np.max(y_padded) + #z = gaussian_filter1d(y_padded, sigma_gaus) + #peaks, _ = find_peaks(z, height=0) + #peaks = peaks - 20 + zneg_rev = np.max(y_padded) - y_padded zneg = np.zeros(len(zneg_rev) + 40) zneg[20 : len(zneg_rev) + 20] = zneg_rev zneg = gaussian_filter1d(zneg, sigma_gaus) - peaks, _ = find_peaks(z, height=0) peaks_neg, _ = find_peaks(zneg, height=0) peaks_neg = peaks_neg - 20 - 20 - peaks = peaks - 20 ##plt.plot(z) ##plt.show() - if contours_main != None: - areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) - x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + cx_main, cy_main = find_center_of_contours(contours_main) + cx_head, cy_head = find_center_of_contours(contours_head) - y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) - y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + peaks_neg_new = np.append(np.insert(peaks_neg, 0, 0), textline_mask.shape[0]) + # offset from bbox of mask + peaks_neg_new += y_ref - if len(contours_header) != None: - areas_header = np.array([cv2.contourArea(contours_header[j]) for j in range(len(contours_header))]) - M_header = [cv2.moments(contours_header[j]) for j in range(len(contours_header))] - cx_header = [(M_header[j]["m10"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] - cy_header = [(M_header[j]["m01"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] - x_min_header = np.array([np.min(contours_header[j][:, 0, 0]) for j in range(len(contours_header))]) - x_max_header = np.array([np.max(contours_header[j][:, 0, 0]) for j in range(len(contours_header))]) - - y_min_header = np.array([np.min(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) - y_max_header = np.array([np.max(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) - # print(cy_main,'mainy') - - peaks_neg_new = [] - peaks_neg_new.append(0 + y_ref) - for iii in range(len(peaks_neg)): - peaks_neg_new.append(peaks_neg[iii] + y_ref) - peaks_neg_new.append(textline_mask.shape[0] + y_ref) - - if len(cy_main) > 0 and np.max(cy_main) > np.max(peaks_neg_new): - cy_main = np.array(cy_main) * (np.max(peaks_neg_new) / np.max(cy_main)) - 10 - if contours_main != None: - indexer_main = np.arange(len(contours_main)) - if contours_main != None: - len_main = len(contours_main) - else: - len_main = 0 - - matrix_of_orders = np.zeros((len(contours_main) + len(contours_header), 5)) - matrix_of_orders[:, 0] = np.arange(len(contours_main) + len(contours_header)) + matrix_of_orders = np.zeros((len(contours_main) + len(contours_head), 5), dtype=int) + matrix_of_orders[:, 0] = np.arange(len(contours_main) + len(contours_head)) matrix_of_orders[: len(contours_main), 1] = 1 matrix_of_orders[len(contours_main) :, 1] = 2 matrix_of_orders[: len(contours_main), 2] = cx_main - matrix_of_orders[len(contours_main) :, 2] = cx_header + matrix_of_orders[len(contours_main) :, 2] = cx_head matrix_of_orders[: len(contours_main), 3] = cy_main - matrix_of_orders[len(contours_main) :, 3] = cy_header + matrix_of_orders[len(contours_main) :, 3] = cy_head matrix_of_orders[: len(contours_main), 4] = np.arange(len(contours_main)) - matrix_of_orders[len(contours_main) :, 4] = np.arange(len(contours_header)) + matrix_of_orders[len(contours_main) :, 4] = np.arange(len(contours_head)) # print(peaks_neg_new,'peaks_neg_new') # print(matrix_of_orders,'matrix_of_orders') @@ -1290,27 +1240,14 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): final_indexers_sorted = [] final_types = [] final_index_type = [] - for i in range(len(peaks_neg_new) - 1): - top = peaks_neg_new[i] - down = peaks_neg_new[i + 1] - indexes_in = matrix_of_orders[:, 0][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] - cxs_in = matrix_of_orders[:, 2][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] - cys_in = matrix_of_orders[:, 3][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] - types_of_text = matrix_of_orders[:, 1][(matrix_of_orders[:, 3] >= top) & - (matrix_of_orders[:, 3] < down)] - index_types_of_text = matrix_of_orders[:, 4][(matrix_of_orders[:, 3] >= top) & - (matrix_of_orders[:, 3] < down)] + for top, bot in pairwise(peaks_neg_new): + indexes_in, types_in, cxs_in, cys_in, typed_indexes_in = \ + matrix_of_orders[(matrix_of_orders[:, 3] >= top) & + (matrix_of_orders[:, 3] < bot)].T sorted_inside = np.argsort(cxs_in) - ind_in_int = indexes_in[sorted_inside] - ind_in_type = types_of_text[sorted_inside] - ind_ind_type = index_types_of_text[sorted_inside] - for j in range(len(ind_in_int)): - final_indexers_sorted.append(int(ind_in_int[j])) - final_types.append(int(ind_in_type[j])) - final_index_type.append(int(ind_ind_type[j])) + final_indexers_sorted.extend(indexes_in[sorted_inside]) + final_types.extend(types_in[sorted_inside]) + final_index_type.extend(typed_indexes_in[sorted_inside]) ##matrix_of_orders[:len_main,4]=final_indexers_sorted[:] From 7387f5a92994bc5c2678be643816e5883f32cfa1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 22:35:40 +0200 Subject: [PATCH 320/374] do_order_of_regions: improve box matching, simplify - when searching for boxes matching contour, be more precise: - avoid heuristic rules ("xmin + 80 within xrange") in favour of exact criteria (contour properly contained in box) - for fallback criterion (nearest centers), also require proper containment of center in box - `order_of_regions`: remove (now) unnecessary (and insufficient) workaround for missing indexes (if boxes are not covering contours exactly) --- src/eynollah/eynollah.py | 185 ++++++++++++++++++--------------- src/eynollah/utils/__init__.py | 14 +-- 2 files changed, 106 insertions(+), 93 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 8351ab6..3194b66 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2518,51 +2518,59 @@ class Eynollah: contours_only_text_parent = np.array(contours_only_text_parent) contours_only_text_parent_h = np.array(contours_only_text_parent_h) boxes = np.array(boxes, dtype=int) # to be on the safe side - cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours( + c_boxes = np.stack((0.5 * boxes[:, 2:4].sum(axis=1), + 0.5 * boxes[:, 0:2].sum(axis=1))) + cx_main, cy_main, mx_main, Mx_main, my_main, My_main, mxy_main = find_new_features_of_contours( contours_only_text_parent) - cx_text_only_h, cy_text_only_h, x_min_text_only_h, _, _, _, y_cor_x_min_main_h = find_new_features_of_contours( + cx_head, cy_head, mx_head, Mx_head, my_head, My_head, mxy_head = find_new_features_of_contours( contours_only_text_parent_h) try: arg_text_con = [] - for ii in range(len(cx_text_only)): + for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (x_min_text_only[ii] + 80 >= boxes[jj][0] and - x_min_text_only[ii] + 80 < boxes[jj][1] and - y_cor_x_min_main[ii] >= boxes[jj][2] and - y_cor_x_min_main[ii] < boxes[jj][3]): + for jj, box in enumerate(boxes): + if (mx_main[ii] >= box[0] and + Mx_main[ii] < box[1] and + my_main[ii] >= box[2] and + My_main[ii] < box[3]): arg_text_con.append(jj) check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) + # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + + # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) + # for box in boxes] + dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) + pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & + (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) + ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) + args_contours = np.arange(len(arg_text_con)) + order_by_con_main = np.zeros(len(arg_text_con)) + arg_text_con_h = [] - for ii in range(len(cx_text_only_h)): + for ii in range(len(contours_only_text_parent_h)): check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (x_min_text_only_h[ii] + 80 >= boxes[jj][0] and - x_min_text_only_h[ii] + 80 < boxes[jj][1] and - y_cor_x_min_main_h[ii] >= boxes[jj][2] and - y_cor_x_min_main_h[ii] < boxes[jj][3]): + for jj, box in enumerate(boxes): + if (mx_head[ii] >= box[0] and + Mx_head[ii] < box[1] and + my_head[ii] >= box[2] and + My_head[ii] < box[3]): arg_text_con_h.append(jj) check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only_h[ii] - boxes[jj][1]) ** 2 + - (cy_text_only_h[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) + # dists_tr_from_box = [math.sqrt((cx_head[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + + # (cy_head[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) + # for box in boxes] + dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_head[ii]], [cx_head[ii]]]), axis=0) + pcontained_in_box = ((boxes[:, 2] <= cy_head[ii]) & (cy_head[ii] < boxes[:, 3]) & + (boxes[:, 0] <= cx_head[ii]) & (cx_head[ii] < boxes[:, 1])) + ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) arg_text_con_h.append(ind_min) - args_contours_h = np.array(range(len(arg_text_con_h))) - + args_contours_h = np.arange(len(arg_text_con_h)) order_by_con_head = np.zeros(len(arg_text_con_h)) - order_by_con_main = np.zeros(len(arg_text_con)) ref_point = 0 order_of_texts_tot = [] @@ -2590,12 +2598,12 @@ class Eynollah: for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for zahler, _ in enumerate(args_contours_box_h): arg_order_v = indexes_sorted_head[zahler] order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji in range(len(id_of_texts)): order_of_texts_tot.append(order_of_texts[jji] + ref_point) @@ -2611,53 +2619,59 @@ class Eynollah: order_text_new = [] for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) except Exception as why: self.logger.error(why) arg_text_con = [] - for ii in range(len(cx_text_only)): + for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (cx_text_only[ii] >= boxes[jj][0] and - cx_text_only[ii] < boxes[jj][1] and - cy_text_only[ii] >= boxes[jj][2] and - cy_text_only[ii] < boxes[jj][3]): + for jj, box in enumerate(boxes): + if (cx_main[ii] >= box[0] and + cx_main[ii] < box[1] and + cy_main[ii] >= box[2] and + cy_main[ii] < box[3]): # this is valid if the center of region identify in which box it is located arg_text_con.append(jj) check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) + # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + + # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) + # for box in boxes] + dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) + pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & + (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) + ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) + args_contours = np.arange(len(arg_text_con)) order_by_con_main = np.zeros(len(arg_text_con)) ############################# head arg_text_con_h = [] - for ii in range(len(cx_text_only_h)): + for ii in range(len(contours_only_text_parent_h)): check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (cx_text_only_h[ii] >= boxes[jj][0] and - cx_text_only_h[ii] < boxes[jj][1] and - cy_text_only_h[ii] >= boxes[jj][2] and - cy_text_only_h[ii] < boxes[jj][3]): + for jj, box in enumerate(boxes): + if (cx_head[ii] >= box[0] and + cx_head[ii] < box[1] and + cy_head[ii] >= box[2] and + cy_head[ii] < box[3]): # this is valid if the center of region identify in which box it is located arg_text_con_h.append(jj) check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only_h[ii] - boxes[jj][1]) ** 2 + - (cy_text_only_h[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) + # dists_tr_from_box = [math.sqrt((cx_head[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + + # (cy_head[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) + # for box in boxes] + dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_head[ii]], [cx_head[ii]]]), axis=0) + pcontained_in_box = ((boxes[:, 2] <= cy_head[ii]) & (cy_head[ii] < boxes[:, 3]) & + (boxes[:, 0] <= cx_head[ii]) & (cx_head[ii] < boxes[:, 1])) + ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) arg_text_con_h.append(ind_min) - args_contours_h = np.array(range(len(arg_text_con_h))) + args_contours_h = np.arange(len(arg_text_con_h)) order_by_con_head = np.zeros(len(arg_text_con_h)) ref_point = 0 @@ -2686,14 +2700,14 @@ class Eynollah: for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for zahler, _ in enumerate(args_contours_box_h): arg_order_v = indexes_sorted_head[zahler] order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - for jji, _ in enumerate(id_of_texts): + for jji in range(len(id_of_texts)): order_of_texts_tot.append(order_of_texts[jji] + ref_point) id_of_texts_tot.append(id_of_texts[jji]) ref_point += len(id_of_texts) @@ -2707,7 +2721,7 @@ class Eynollah: order_text_new = [] for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) self.logger.debug("exit do_order_of_regions_full_layout") return order_text_new, id_of_texts_tot @@ -2719,28 +2733,33 @@ class Eynollah: contours_only_text_parent = np.array(contours_only_text_parent) contours_only_text_parent_h = np.array(contours_only_text_parent_h) boxes = np.array(boxes, dtype=int) # to be on the safe side - cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours( + c_boxes = np.stack((0.5 * boxes[:, 2:4].sum(axis=1), + 0.5 * boxes[:, 0:2].sum(axis=1))) + cx_main, cy_main, mx_main, Mx_main, my_main, My_main, mxy_main = find_new_features_of_contours( contours_only_text_parent) try: arg_text_con = [] - for ii in range(len(cx_text_only)): + for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (x_min_text_only[ii] + 80 >= boxes[jj][0] and - x_min_text_only[ii] + 80 < boxes[jj][1] and - y_cor_x_min_main[ii] >= boxes[jj][2] and - y_cor_x_min_main[ii] < boxes[jj][3]): + for jj, box in enumerate(boxes): + if (mx_main[ii] >= box[0] and + Mx_main[ii] < box[1] and + my_main[ii] >= box[2] and + My_main[ii] < box[3]): arg_text_con.append(jj) check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) + # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + + # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) + # for box in boxes] + dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) + pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & + (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) + ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) + args_contours = np.arange(len(arg_text_con)) order_by_con_main = np.zeros(len(arg_text_con)) ref_point = 0 @@ -2766,7 +2785,7 @@ class Eynollah: for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji, _ in enumerate(id_of_texts): order_of_texts_tot.append(order_of_texts[jji] + ref_point) @@ -2779,29 +2798,29 @@ class Eynollah: order_text_new = [] for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) except Exception as why: self.logger.error(why) arg_text_con = [] - for ii in range(len(cx_text_only)): + for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (cx_text_only[ii] >= boxes[jj][0] and - cx_text_only[ii] < boxes[jj][1] and - cy_text_only[ii] >= boxes[jj][2] and - cy_text_only[ii] < boxes[jj][3]): + for jj, box in enumerate(boxes): + if (cx_main[ii] >= box[0] and + cx_main[ii] < box[1] and + cy_main[ii] >= box[2] and + cy_main[ii] < box[3]): # this is valid if the center of region identify in which box it is located arg_text_con.append(jj) check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) + dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) + pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & + (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) + ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) + arg_text_con[ii] = ind_min + args_contours = np.arange(len(contours_only_text_parent)) order_by_con_main = np.zeros(len(arg_text_con)) ref_point = 0 @@ -2829,7 +2848,7 @@ class Eynollah: for zahler, _ in enumerate(args_contours_box): arg_order_v = indexes_sorted_main[zahler] order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji, _ in enumerate(id_of_texts): order_of_texts_tot.append(order_of_texts[jji] + ref_point) @@ -2843,7 +2862,7 @@ class Eynollah: order_text_new = [] for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) self.logger.debug("exit do_order_of_regions_no_full_layout") return order_text_new, id_of_texts_tot diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 92da14a..6e5afd4 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -1222,6 +1222,8 @@ def order_of_regions(textline_mask, contours_main, contours_head, y_ref): # offset from bbox of mask peaks_neg_new += y_ref + # assert not len(cy_main) or np.min(peaks_neg_new) <= np.min(cy_main) and np.max(cy_main) <= np.max(peaks_neg_new) + # assert not len(cy_head) or np.min(peaks_neg_new) <= np.min(cy_head) and np.max(cy_head) <= np.max(peaks_neg_new) matrix_of_orders = np.zeros((len(contours_main) + len(contours_head), 5), dtype=int) matrix_of_orders[:, 0] = np.arange(len(contours_main) + len(contours_head)) @@ -1251,16 +1253,8 @@ def order_of_regions(textline_mask, contours_main, contours_head, y_ref): ##matrix_of_orders[:len_main,4]=final_indexers_sorted[:] - # This fix is applied if the sum of the lengths of contours and contours_h - # does not match final_indexers_sorted. However, this is not the optimal solution.. - if len(cy_main) + len(cy_header) == len(final_index_type): - pass - else: - indexes_missed = set(np.arange(len(cy_main) + len(cy_header))) - set(final_indexers_sorted) - for ind_missed in indexes_missed: - final_indexers_sorted.append(ind_missed) - final_types.append(1) - final_index_type.append(ind_missed) + # assert len(final_indexers_sorted) == len(contours_main) + len(contours_head) + # assert not len(final_indexers_sorted) or max(final_index_type) == max(len(contours_main) return np.array(final_indexers_sorted), np.array(final_types), np.array(final_index_type) From e9bb62bd86747dabd5cd6fb5f67a36547c5c626d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 2 Oct 2025 23:44:00 +0200 Subject: [PATCH 321/374] do_order_of_regions: simplify - avoid loops in favour of array processing --- src/eynollah/eynollah.py | 158 ++++++++++++++++----------------------- 1 file changed, 64 insertions(+), 94 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 3194b66..6a3fd1e 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2526,7 +2526,7 @@ class Eynollah: contours_only_text_parent_h) try: - arg_text_con = [] + arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): @@ -2534,7 +2534,7 @@ class Eynollah: Mx_main[ii] < box[1] and my_main[ii] >= box[2] and My_main[ii] < box[3]): - arg_text_con.append(jj) + arg_text_con_main[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: @@ -2545,11 +2545,11 @@ class Eynollah: pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con.append(ind_min) - args_contours = np.arange(len(arg_text_con)) - order_by_con_main = np.zeros(len(arg_text_con)) + arg_text_con_main[ii] = ind_min + args_contours_main = np.arange(len(contours_only_text_parent)) + order_by_con_main = np.zeros_like(arg_text_con_main) - arg_text_con_h = [] + arg_text_con_head = np.zeros(len(contours_only_text_parent_h), dtype=int) for ii in range(len(contours_only_text_parent_h)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): @@ -2557,7 +2557,7 @@ class Eynollah: Mx_head[ii] < box[1] and my_head[ii] >= box[2] and My_head[ii] < box[3]): - arg_text_con_h.append(jj) + arg_text_con_head[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: @@ -2568,9 +2568,9 @@ class Eynollah: pcontained_in_box = ((boxes[:, 2] <= cy_head[ii]) & (cy_head[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_head[ii]) & (cx_head[ii] < boxes[:, 1])) ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con_h.append(ind_min) - args_contours_h = np.arange(len(arg_text_con_h)) - order_by_con_head = np.zeros(len(arg_text_con_h)) + arg_text_con_head[ii] = ind_min + args_contours_head = np.arange(len(contours_only_text_parent_h)) + order_by_con_head = np.zeros_like(arg_text_con_head) ref_point = 0 order_of_texts_tot = [] @@ -2578,10 +2578,10 @@ class Eynollah: for iij, box in enumerate(boxes): ys = slice(*box[2:4]) xs = slice(*box[0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] - con_inter_box = contours_only_text_parent[args_contours_box] - con_inter_box_h = contours_only_text_parent_h[args_contours_box_h] + args_contours_box_main = args_contours_main[arg_text_con_main == iij] + args_contours_box_head = args_contours_head[arg_text_con_head == iij] + con_inter_box = contours_only_text_parent[args_contours_box_main] + con_inter_box_h = contours_only_text_parent_h[args_contours_box_head] indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) @@ -2595,14 +2595,14 @@ class Eynollah: indexes_sorted_head = indexes_sorted[kind_of_texts_sorted == 2] indexes_by_type_head = index_by_kind_sorted[kind_of_texts_sorted == 2] - for zahler, _ in enumerate(args_contours_box): + for zahler, _ in enumerate(args_contours_box_main): arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ + order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - for zahler, _ in enumerate(args_contours_box_h): + for zahler, _ in enumerate(args_contours_box_head): arg_order_v = indexes_sorted_head[zahler] - order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = \ + order_by_con_head[args_contours_box_head[indexes_by_type_head[zahler]]] = \ np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji in range(len(id_of_texts)): @@ -2610,20 +2610,13 @@ class Eynollah: id_of_texts_tot.append(id_of_texts[jji]) ref_point += len(id_of_texts) - order_of_texts_tot = [] - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - for tj1 in range(len(contours_only_text_parent_h)): - order_of_texts_tot.append(int(order_by_con_head[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) + order_of_texts_tot = np.concatenate((order_by_con_main, + order_by_con_head)) + order_text_new = np.argsort(order_of_texts_tot) except Exception as why: self.logger.error(why) - arg_text_con = [] + arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): @@ -2632,10 +2625,9 @@ class Eynollah: cy_main[ii] >= box[2] and cy_main[ii] < box[3]): # this is valid if the center of region identify in which box it is located - arg_text_con.append(jj) + arg_text_con_main[ii] = jj check_if_textregion_located_in_a_box = True break - if not check_if_textregion_located_in_a_box: # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) @@ -2644,13 +2636,11 @@ class Eynollah: pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con.append(ind_min) - args_contours = np.arange(len(arg_text_con)) - order_by_con_main = np.zeros(len(arg_text_con)) + arg_text_con_main[ii] = ind_min + args_contours_main = np.arange(len(contours_only_text_parent)) + order_by_con_main = np.zeros_like(arg_text_con_main) - ############################# head - - arg_text_con_h = [] + arg_text_con_head = np.zeros(len(contours_only_text_parent_h), dtype=int) for ii in range(len(contours_only_text_parent_h)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): @@ -2659,7 +2649,7 @@ class Eynollah: cy_head[ii] >= box[2] and cy_head[ii] < box[3]): # this is valid if the center of region identify in which box it is located - arg_text_con_h.append(jj) + arg_text_con_head[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: @@ -2670,9 +2660,9 @@ class Eynollah: pcontained_in_box = ((boxes[:, 2] <= cy_head[ii]) & (cy_head[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_head[ii]) & (cx_head[ii] < boxes[:, 1])) ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con_h.append(ind_min) - args_contours_h = np.arange(len(arg_text_con_h)) - order_by_con_head = np.zeros(len(arg_text_con_h)) + arg_text_con_head[ii] = ind_min + args_contours_head = np.arange(len(contours_only_text_parent_h)) + order_by_con_head = np.zeros_like(arg_text_con_head) ref_point = 0 order_of_texts_tot = [] @@ -2680,10 +2670,10 @@ class Eynollah: for iij, box in enumerate(boxes): ys = slice(*box[2:4]) xs = slice(*box[0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] - con_inter_box = contours_only_text_parent[args_contours_box] - con_inter_box_h = contours_only_text_parent_h[args_contours_box_h] + args_contours_box_main = args_contours_main[arg_text_con_main == iij] + args_contours_box_head = args_contours_head[arg_text_con_head == iij] + con_inter_box = contours_only_text_parent[args_contours_box_main] + con_inter_box_h = contours_only_text_parent_h[args_contours_box_head] indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) @@ -2697,14 +2687,14 @@ class Eynollah: indexes_sorted_head = indexes_sorted[kind_of_texts_sorted == 2] indexes_by_type_head = index_by_kind_sorted[kind_of_texts_sorted == 2] - for zahler, _ in enumerate(args_contours_box): + for zahler, _ in enumerate(args_contours_box_main): arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ + order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - for zahler, _ in enumerate(args_contours_box_h): + for zahler, _ in enumerate(args_contours_box_head): arg_order_v = indexes_sorted_head[zahler] - order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = \ + order_by_con_head[args_contours_box_head[indexes_by_type_head[zahler]]] = \ np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji in range(len(id_of_texts)): @@ -2712,16 +2702,9 @@ class Eynollah: id_of_texts_tot.append(id_of_texts[jji]) ref_point += len(id_of_texts) - order_of_texts_tot = [] - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - for tj1 in range(len(contours_only_text_parent_h)): - order_of_texts_tot.append(int(order_by_con_head[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) + order_of_texts_tot = np.concatenate((order_by_con_main, + order_by_con_head)) + order_text_new = np.argsort(order_of_texts_tot) self.logger.debug("exit do_order_of_regions_full_layout") return order_text_new, id_of_texts_tot @@ -2739,7 +2722,7 @@ class Eynollah: contours_only_text_parent) try: - arg_text_con = [] + arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): @@ -2747,7 +2730,7 @@ class Eynollah: Mx_main[ii] < box[1] and my_main[ii] >= box[2] and My_main[ii] < box[3]): - arg_text_con.append(jj) + arg_text_con_main[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: @@ -2758,9 +2741,9 @@ class Eynollah: pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con.append(ind_min) - args_contours = np.arange(len(arg_text_con)) - order_by_con_main = np.zeros(len(arg_text_con)) + arg_text_con_main[ii] = ind_min + args_contours_main = np.arange(len(contours_only_text_parent)) + order_by_con_main = np.zeros_like(arg_text_con_main) ref_point = 0 order_of_texts_tot = [] @@ -2768,8 +2751,8 @@ class Eynollah: for iij, box in enumerate(boxes): ys = slice(*box[2:4]) xs = slice(*box[0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - con_inter_box = contours_only_text_parent[args_contours_box] + args_contours_box_main = args_contours_main[arg_text_con_main == iij] + con_inter_box = contours_only_text_parent[args_contours_box_main] con_inter_box_h = [] indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( @@ -2782,9 +2765,9 @@ class Eynollah: indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] - for zahler, _ in enumerate(args_contours_box): + for zahler, _ in enumerate(args_contours_box_main): arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ + order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji, _ in enumerate(id_of_texts): @@ -2792,17 +2775,12 @@ class Eynollah: id_of_texts_tot.append(id_of_texts[jji]) ref_point += len(id_of_texts) - order_of_texts_tot = [] - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) + order_of_texts_tot = order_by_con_main + order_text_new = np.argsort(order_of_texts_tot) except Exception as why: self.logger.error(why) - arg_text_con = [] + arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): @@ -2811,7 +2789,7 @@ class Eynollah: cy_main[ii] >= box[2] and cy_main[ii] < box[3]): # this is valid if the center of region identify in which box it is located - arg_text_con.append(jj) + arg_text_con_main[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: @@ -2819,9 +2797,9 @@ class Eynollah: pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con[ii] = ind_min - args_contours = np.arange(len(contours_only_text_parent)) - order_by_con_main = np.zeros(len(arg_text_con)) + arg_text_con_main[ii] = ind_min + args_contours_main = np.arange(len(contours_only_text_parent)) + order_by_con_main = np.zeros_like(arg_text_con_main) ref_point = 0 order_of_texts_tot = [] @@ -2829,11 +2807,9 @@ class Eynollah: for iij, box in enumerate(boxes): ys = slice(*box[2:4]) xs = slice(*box[0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - con_inter_box = [] + args_contours_box_main = args_contours_main[arg_text_con_main == iij] + con_inter_box = contours_only_text_parent[args_contours_box_main] con_inter_box_h = [] - for i in range(len(args_contours_box)): - con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) @@ -2845,9 +2821,9 @@ class Eynollah: indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] - for zahler, _ in enumerate(args_contours_box): + for zahler, _ in enumerate(args_contours_box_main): arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ + order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ np.flatnonzero(indexes_sorted == arg_order_v) + ref_point for jji, _ in enumerate(id_of_texts): @@ -2855,14 +2831,8 @@ class Eynollah: id_of_texts_tot.append(id_of_texts[jji]) ref_point += len(id_of_texts) - order_of_texts_tot = [] - - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.flatnonzero(np.array(order_of_texts_tot) == iii)) + order_of_texts_tot = order_by_con_main + order_text_new = np.argsort(order_of_texts_tot) self.logger.debug("exit do_order_of_regions_no_full_layout") return order_text_new, id_of_texts_tot From e674ea08f383de0c87f950be153fc954c3b4308e Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 3 Oct 2025 00:59:25 +0200 Subject: [PATCH 322/374] do_order_of_regions: drop redundant no/full_layout (`_no_full_layout` is the same copied code as `_full_layout`; the latter runs just the same if passed an empty list for headings) --- src/eynollah/eynollah.py | 141 ++------------------------------------ src/eynollah/utils/xml.py | 4 +- 2 files changed, 6 insertions(+), 139 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6a3fd1e..629b001 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2511,10 +2511,10 @@ class Eynollah: self.logger.debug("exit get_regions_from_xy_2models") return text_regions_p_true, erosion_hurts, polygons_seplines - def do_order_of_regions_full_layout( + def do_order_of_regions( self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): - self.logger.debug("enter do_order_of_regions_full_layout") + self.logger.debug("enter do_order_of_regions") contours_only_text_parent = np.array(contours_only_text_parent) contours_only_text_parent_h = np.array(contours_only_text_parent_h) boxes = np.array(boxes, dtype=int) # to be on the safe side @@ -2706,135 +2706,7 @@ class Eynollah: order_by_con_head)) order_text_new = np.argsort(order_of_texts_tot) - self.logger.debug("exit do_order_of_regions_full_layout") - return order_text_new, id_of_texts_tot - - def do_order_of_regions_no_full_layout( - self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): - - self.logger.debug("enter do_order_of_regions_no_full_layout") - contours_only_text_parent = np.array(contours_only_text_parent) - contours_only_text_parent_h = np.array(contours_only_text_parent_h) - boxes = np.array(boxes, dtype=int) # to be on the safe side - c_boxes = np.stack((0.5 * boxes[:, 2:4].sum(axis=1), - 0.5 * boxes[:, 0:2].sum(axis=1))) - cx_main, cy_main, mx_main, Mx_main, my_main, My_main, mxy_main = find_new_features_of_contours( - contours_only_text_parent) - - try: - arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) - for ii in range(len(contours_only_text_parent)): - check_if_textregion_located_in_a_box = False - for jj, box in enumerate(boxes): - if (mx_main[ii] >= box[0] and - Mx_main[ii] < box[1] and - my_main[ii] >= box[2] and - My_main[ii] < box[3]): - arg_text_con_main[ii] = jj - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + - # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) - # for box in boxes] - dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) - pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & - (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) - ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con_main[ii] = ind_min - args_contours_main = np.arange(len(contours_only_text_parent)) - order_by_con_main = np.zeros_like(arg_text_con_main) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij, box in enumerate(boxes): - ys = slice(*box[2:4]) - xs = slice(*box[0:2]) - args_contours_box_main = args_contours_main[arg_text_con_main == iij] - con_inter_box = contours_only_text_parent[args_contours_box_main] - con_inter_box_h = [] - - indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] - indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] - - for zahler, _ in enumerate(args_contours_box_main): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ - np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - - for jji, _ in enumerate(id_of_texts): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = order_by_con_main - order_text_new = np.argsort(order_of_texts_tot) - - except Exception as why: - self.logger.error(why) - arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) - for ii in range(len(contours_only_text_parent)): - check_if_textregion_located_in_a_box = False - for jj, box in enumerate(boxes): - if (cx_main[ii] >= box[0] and - cx_main[ii] < box[1] and - cy_main[ii] >= box[2] and - cy_main[ii] < box[3]): - # this is valid if the center of region identify in which box it is located - arg_text_con_main[ii] = jj - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) - pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & - (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) - ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con_main[ii] = ind_min - args_contours_main = np.arange(len(contours_only_text_parent)) - order_by_con_main = np.zeros_like(arg_text_con_main) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij, box in enumerate(boxes): - ys = slice(*box[2:4]) - xs = slice(*box[0:2]) - args_contours_box_main = args_contours_main[arg_text_con_main == iij] - con_inter_box = contours_only_text_parent[args_contours_box_main] - con_inter_box_h = [] - - indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] - indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] - - for zahler, _ in enumerate(args_contours_box_main): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ - np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - - for jji, _ in enumerate(id_of_texts): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = order_by_con_main - order_text_new = np.argsort(order_of_texts_tot) - - self.logger.debug("exit do_order_of_regions_no_full_layout") + self.logger.debug("exit do_order_of_regions") return order_text_new, id_of_texts_tot def check_iou_of_bounding_box_and_contour_for_tables( @@ -3081,11 +2953,6 @@ class Eynollah: image_revised_last[int(y_min_tab_col1[ijv]):int(y_max_tab_col1[ijv]),:,:]=pixel_table return image_revised_last - def do_order_of_regions(self, *args, **kwargs): - if self.full_layout: - return self.do_order_of_regions_full_layout(*args, **kwargs) - return self.do_order_of_regions_no_full_layout(*args, **kwargs) - def get_tables_from_model(self, img, num_col_classifier): img_org = np.copy(img) img_height_h = img_org.shape[0] @@ -5170,7 +5037,7 @@ class Eynollah: return pcgts - contours_only_text_parent_h = None + contours_only_text_parent_h = [] self.logger.info("Step 4/5: Reading Order Detection") if self.reading_order_machine_based: diff --git a/src/eynollah/utils/xml.py b/src/eynollah/utils/xml.py index a61dadb..88d1df8 100644 --- a/src/eynollah/utils/xml.py +++ b/src/eynollah/utils/xml.py @@ -57,8 +57,8 @@ def xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_margina og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=id_marginal)) region_counter.inc('region') - for idx_textregion, _ in enumerate(order_of_texts): - og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=region_counter.region_id(order_of_texts[idx_textregion] + 1))) + for idx_textregion in order_of_texts: + og.add_RegionRefIndexed(RegionRefIndexedType(index=str(region_counter.get('region')), regionRef=region_counter.region_id(idx_textregion + 1))) region_counter.inc('region') for id_marginal in id_of_marginalia_right: From 29b4527bdebf6583f32b8801aed26f6ae70d25c7 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 3 Oct 2025 02:06:08 +0200 Subject: [PATCH 323/374] do_order_of_regions: simplify - remove duplicate code via inline def for the try-catch --- src/eynollah/eynollah.py | 127 +++++++-------------------------------- 1 file changed, 22 insertions(+), 105 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 629b001..bb3d1bf 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2525,22 +2525,23 @@ class Eynollah: cx_head, cy_head, mx_head, Mx_head, my_head, My_head, mxy_head = find_new_features_of_contours( contours_only_text_parent_h) - try: + def match_boxes(only_centers: bool): arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) for ii in range(len(contours_only_text_parent)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): - if (mx_main[ii] >= box[0] and - Mx_main[ii] < box[1] and - my_main[ii] >= box[2] and - My_main[ii] < box[3]): + if ((cx_main[ii] >= box[0] and + cx_main[ii] < box[1] and + cy_main[ii] >= box[2] and + cy_main[ii] < box[3]) if only_centers else + (mx_main[ii] >= box[0] and + Mx_main[ii] < box[1] and + my_main[ii] >= box[2] and + My_main[ii] < box[3])): arg_text_con_main[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + - # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) - # for box in boxes] dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) @@ -2553,17 +2554,18 @@ class Eynollah: for ii in range(len(contours_only_text_parent_h)): check_if_textregion_located_in_a_box = False for jj, box in enumerate(boxes): - if (mx_head[ii] >= box[0] and - Mx_head[ii] < box[1] and - my_head[ii] >= box[2] and - My_head[ii] < box[3]): + if ((cx_head[ii] >= box[0] and + cx_head[ii] < box[1] and + cy_head[ii] >= box[2] and + cy_head[ii] < box[3]) if only_centers else + (mx_head[ii] >= box[0] and + Mx_head[ii] < box[1] and + my_head[ii] >= box[2] and + My_head[ii] < box[3])): arg_text_con_head[ii] = jj check_if_textregion_located_in_a_box = True break if not check_if_textregion_located_in_a_box: - # dists_tr_from_box = [math.sqrt((cx_head[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + - # (cy_head[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) - # for box in boxes] dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_head[ii]], [cx_head[ii]]]), axis=0) pcontained_in_box = ((boxes[:, 2] <= cy_head[ii]) & (cy_head[ii] < boxes[:, 3]) & (boxes[:, 0] <= cx_head[ii]) & (cx_head[ii] < boxes[:, 1])) @@ -2613,101 +2615,16 @@ class Eynollah: order_of_texts_tot = np.concatenate((order_by_con_main, order_by_con_head)) order_text_new = np.argsort(order_of_texts_tot) + return order_text_new, id_of_texts_tot + try: + results = match_boxes(False) except Exception as why: self.logger.error(why) - arg_text_con_main = np.zeros(len(contours_only_text_parent), dtype=int) - for ii in range(len(contours_only_text_parent)): - check_if_textregion_located_in_a_box = False - for jj, box in enumerate(boxes): - if (cx_main[ii] >= box[0] and - cx_main[ii] < box[1] and - cy_main[ii] >= box[2] and - cy_main[ii] < box[3]): - # this is valid if the center of region identify in which box it is located - arg_text_con_main[ii] = jj - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - # dists_tr_from_box = [math.sqrt((cx_main[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + - # (cy_main[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) - # for box in boxes] - dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_main[ii]], [cx_main[ii]]]), axis=0) - pcontained_in_box = ((boxes[:, 2] <= cy_main[ii]) & (cy_main[ii] < boxes[:, 3]) & - (boxes[:, 0] <= cx_main[ii]) & (cx_main[ii] < boxes[:, 1])) - ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con_main[ii] = ind_min - args_contours_main = np.arange(len(contours_only_text_parent)) - order_by_con_main = np.zeros_like(arg_text_con_main) - - arg_text_con_head = np.zeros(len(contours_only_text_parent_h), dtype=int) - for ii in range(len(contours_only_text_parent_h)): - check_if_textregion_located_in_a_box = False - for jj, box in enumerate(boxes): - if (cx_head[ii] >= box[0] and - cx_head[ii] < box[1] and - cy_head[ii] >= box[2] and - cy_head[ii] < box[3]): - # this is valid if the center of region identify in which box it is located - arg_text_con_head[ii] = jj - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - # dists_tr_from_box = [math.sqrt((cx_head[ii] - 0.5 * box[1] - 0.5 * box[0]) ** 2 + - # (cy_head[ii] - 0.5 * box[3] - 0.5 * box[2]) ** 2) - # for box in boxes] - dists_tr_from_box = np.linalg.norm(c_boxes - np.array([[cy_head[ii]], [cx_head[ii]]]), axis=0) - pcontained_in_box = ((boxes[:, 2] <= cy_head[ii]) & (cy_head[ii] < boxes[:, 3]) & - (boxes[:, 0] <= cx_head[ii]) & (cx_head[ii] < boxes[:, 1])) - ind_min = np.argmin(np.ma.masked_array(dists_tr_from_box, ~pcontained_in_box)) - arg_text_con_head[ii] = ind_min - args_contours_head = np.arange(len(contours_only_text_parent_h)) - order_by_con_head = np.zeros_like(arg_text_con_head) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij, box in enumerate(boxes): - ys = slice(*box[2:4]) - xs = slice(*box[0:2]) - args_contours_box_main = args_contours_main[arg_text_con_main == iij] - args_contours_box_head = args_contours_head[arg_text_con_head == iij] - con_inter_box = contours_only_text_parent[args_contours_box_main] - con_inter_box_h = contours_only_text_parent_h[args_contours_box_head] - - indexes_sorted, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, box[2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = indexes_sorted[kind_of_texts_sorted == 1] - indexes_by_type_main = index_by_kind_sorted[kind_of_texts_sorted == 1] - indexes_sorted_head = indexes_sorted[kind_of_texts_sorted == 2] - indexes_by_type_head = index_by_kind_sorted[kind_of_texts_sorted == 2] - - for zahler, _ in enumerate(args_contours_box_main): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box_main[indexes_by_type_main[zahler]]] = \ - np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - - for zahler, _ in enumerate(args_contours_box_head): - arg_order_v = indexes_sorted_head[zahler] - order_by_con_head[args_contours_box_head[indexes_by_type_head[zahler]]] = \ - np.flatnonzero(indexes_sorted == arg_order_v) + ref_point - - for jji in range(len(id_of_texts)): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = np.concatenate((order_by_con_main, - order_by_con_head)) - order_text_new = np.argsort(order_of_texts_tot) + results = match_boxes(True) self.logger.debug("exit do_order_of_regions") - return order_text_new, id_of_texts_tot + return results def check_iou_of_bounding_box_and_contour_for_tables( self, layout, table_prediction_early, pixel_table, num_col_classifier): From d774a23daa80cad0baa16dc4b41e93b93bca39bf Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sun, 5 Oct 2025 02:18:17 +0200 Subject: [PATCH 324/374] matching deskewed text region contours with predicted: simplify - avoid loops in favour of array processing - improve readability and identifiers --- src/eynollah/eynollah.py | 108 +++++++++++++++------------------------ 1 file changed, 40 insertions(+), 68 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index bb3d1bf..dd6172a 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4559,27 +4559,16 @@ class Eynollah: areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) #self.logger.info('areas_cnt_text %s', areas_cnt_text) - contours_biggest = contours_only_text_parent[np.argmax(areas_cnt_text)] - contours_only_text_parent = [c for jz, c in enumerate(contours_only_text_parent) - if areas_cnt_text[jz] > MIN_AREA_REGION] - areas_cnt_text_parent = [area for area in areas_cnt_text if area > MIN_AREA_REGION] + contour0 = contours_only_text_parent[np.argmax(areas_cnt_text)] + contours_only_text_parent = np.array(contours_only_text_parent)[areas_cnt_text > MIN_AREA_REGION] + areas_cnt_text_parent = areas_cnt_text[areas_cnt_text > MIN_AREA_REGION] + index_con_parents = np.argsort(areas_cnt_text_parent) + contours_only_text_parent = contours_only_text_parent[index_con_parents] + areas_cnt_text_parent = areas_cnt_text_parent[index_con_parents] - contours_only_text_parent = self.return_list_of_contours_with_desired_order( - contours_only_text_parent, index_con_parents) - - ##try: - ##contours_only_text_parent = \ - ##list(np.array(contours_only_text_parent,dtype=object)[index_con_parents]) - ##except: - ##contours_only_text_parent = \ - ##list(np.array(contours_only_text_parent,dtype=np.int32)[index_con_parents]) - ##areas_cnt_text_parent = list(np.array(areas_cnt_text_parent)[index_con_parents]) - areas_cnt_text_parent = self.return_list_of_contours_with_desired_order( - areas_cnt_text_parent, index_con_parents) - - cx_bigest_big, cy_biggest_big = find_center_of_contours([contours_biggest]) - cx_bigest, cy_biggest = find_center_of_contours(contours_only_text_parent) + center0 = np.stack(find_center_of_contours([contour0])) # [2, 1] + centers = np.stack(find_center_of_contours(contours_only_text_parent)) # [2, N] if np.abs(slope_deskew) >= SLOPE_THRESHOLD: contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) @@ -4588,65 +4577,48 @@ class Eynollah: areas_cnt_text_d = np.array([cv2.contourArea(c) for c in contours_only_text_parent_d]) areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) - if len(areas_cnt_text_d)>0: - contours_biggest_d = contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] + if len(contours_only_text_parent_d): + contour0_d = contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] index_con_parents_d = np.argsort(areas_cnt_text_d) - contours_only_text_parent_d = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d, index_con_parents_d) - #try: - #contours_only_text_parent_d = \ - #list(np.array(contours_only_text_parent_d,dtype=object)[index_con_parents_d]) - #except: - #contours_only_text_parent_d = \ - #list(np.array(contours_only_text_parent_d,dtype=np.int32)[index_con_parents_d]) - #areas_cnt_text_d = list(np.array(areas_cnt_text_d)[index_con_parents_d]) - areas_cnt_text_d = self.return_list_of_contours_with_desired_order( - areas_cnt_text_d, index_con_parents_d) + contours_only_text_parent_d = np.array(contours_only_text_parent_d)[index_con_parents_d] + # rs: should be the same, no? + assert np.all(contour0_d == contours_only_text_parent_d[-1]), (np.argmax(areas_cnt_text_d), index_con_parents_d[-1]) + areas_cnt_text_d = areas_cnt_text_d[index_con_parents_d] - cx_bigest_d_big, cy_biggest_d_big = find_center_of_contours([contours_biggest_d]) - cx_bigest_d, cy_biggest_d = find_center_of_contours(contours_only_text_parent_d) - try: - if len(cx_bigest_d) >= 5: - cx_bigest_d_last5 = cx_bigest_d[-5:] - cy_biggest_d_last5 = cy_biggest_d[-5:] - dists_d = [math.sqrt((cx_bigest_big[0] - cx_bigest_d_last5[j]) ** 2 + - (cy_biggest_big[0] - cy_biggest_d_last5[j]) ** 2) - for j in range(len(cy_biggest_d_last5))] - ind_largest = len(cx_bigest_d) -5 + np.argmin(dists_d) - else: - cx_bigest_d_last5 = cx_bigest_d[-len(cx_bigest_d):] - cy_biggest_d_last5 = cy_biggest_d[-len(cx_bigest_d):] - dists_d = [math.sqrt((cx_bigest_big[0]-cx_bigest_d_last5[j])**2 + - (cy_biggest_big[0]-cy_biggest_d_last5[j])**2) - for j in range(len(cy_biggest_d_last5))] - ind_largest = len(cx_bigest_d) - len(cx_bigest_d) + np.argmin(dists_d) - - cx_bigest_d_big[0] = cx_bigest_d[ind_largest] - cy_biggest_d_big[0] = cy_biggest_d[ind_largest] - except Exception as why: - self.logger.error(str(why)) + center0_d = np.stack(find_center_of_contours([contour0_d])) # [2, 1] + centers_d = np.stack(find_center_of_contours(contours_only_text_parent_d)) # [2, N] + # rs: should be the same, no? + assert center0_d[0,0] == centers_d[0,-1] and center0_d[1,0] == centers_d[1,-1] + last5_centers_d = centers_d[:, -5:] + dists_d = np.linalg.norm(center0 - last5_centers_d, axis=0) + ind_largest = len(contours_only_text_parent_d) - last5_centers_d.shape[1] + np.argmin(dists_d) + center0_d[:, 0] = centers_d[:, ind_largest] + # order new contours the same way as the undeskewed contours + # (by calculating the offset of the largest contours, respectively, + # of the new and undeskewed image; then for each contour, + # finding the closest new contour, with proximity calculated + # as distance of their centers modulo offset vector) (h, w) = text_only.shape[:2] center = (w // 2.0, h // 2.0) M = cv2.getRotationMatrix2D(center, slope_deskew, 1.0) M_22 = np.array(M)[:2, :2] - p_big = np.dot(M_22, [cx_bigest_big, cy_biggest_big]) - x_diff = p_big[0] - cx_bigest_d_big - y_diff = p_big[1] - cy_biggest_d_big + p0 = np.dot(M_22, center0) # [2, 1] + offset = p0 - center0_d # [2, 1] + # img2 = np.zeros(text_only_d.shape[:2], dtype=np.uint8) contours_only_text_parent_d_ordered = [] for i in range(len(contours_only_text_parent)): - p = np.dot(M_22, [cx_bigest[i], cy_biggest[i]]) - p[0] = p[0] - x_diff[0] - p[1] = p[1] - y_diff[0] - dists = [math.sqrt((p[0] - cx_bigest_d[j]) ** 2 + - (p[1] - cy_biggest_d[j]) ** 2) - for j in range(len(cx_bigest_d))] - contours_only_text_parent_d_ordered.append(contours_only_text_parent_d[np.argmin(dists)]) - # img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) - # img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d[np.argmin(dists)]] ,color=(1,1,1)) - # plt.imshow(img2[:,:,0]) - # plt.show() + p = np.dot(M_22, centers[:, i:i+1]) # [2, 1] + p -= offset + dists = np.linalg.norm(p - centers_d, axis=0) + contours_only_text_parent_d_ordered.append( + contours_only_text_parent_d[np.argmin(dists)]) + # cv2.fillPoly(img2, pts=[contours_only_text_parent_d[np.argmin(dists)]], color=i + 1) + # plt.imshow(img2) + # plt.show() + # rs: what about the remaining contours_only_text_parent_d? + # rs: what about duplicates? else: contours_only_text_parent_d_ordered = [] contours_only_text_parent_d = [] From 73e5a1def8489f6bf022e696f010d4c852ff685b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sun, 5 Oct 2025 02:33:03 +0200 Subject: [PATCH 325/374] matching deskewed text region contours with predicted: simplify - (no need for argmax if already sorted) --- src/eynollah/eynollah.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index dd6172a..46437f0 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4559,7 +4559,6 @@ class Eynollah: areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) #self.logger.info('areas_cnt_text %s', areas_cnt_text) - contour0 = contours_only_text_parent[np.argmax(areas_cnt_text)] contours_only_text_parent = np.array(contours_only_text_parent)[areas_cnt_text > MIN_AREA_REGION] areas_cnt_text_parent = areas_cnt_text[areas_cnt_text > MIN_AREA_REGION] @@ -4567,9 +4566,11 @@ class Eynollah: contours_only_text_parent = contours_only_text_parent[index_con_parents] areas_cnt_text_parent = areas_cnt_text_parent[index_con_parents] - center0 = np.stack(find_center_of_contours([contour0])) # [2, 1] centers = np.stack(find_center_of_contours(contours_only_text_parent)) # [2, N] + contour0 = contours_only_text_parent[-1] + center0 = centers[:, -1:] # [2, 1] + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) contours_only_text_parent_d = return_parent_contours(contours_only_text_d, hir_on_text_d) @@ -4578,17 +4579,15 @@ class Eynollah: areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) if len(contours_only_text_parent_d): - contour0_d = contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] index_con_parents_d = np.argsort(areas_cnt_text_d) contours_only_text_parent_d = np.array(contours_only_text_parent_d)[index_con_parents_d] - # rs: should be the same, no? - assert np.all(contour0_d == contours_only_text_parent_d[-1]), (np.argmax(areas_cnt_text_d), index_con_parents_d[-1]) areas_cnt_text_d = areas_cnt_text_d[index_con_parents_d] - center0_d = np.stack(find_center_of_contours([contour0_d])) # [2, 1] centers_d = np.stack(find_center_of_contours(contours_only_text_parent_d)) # [2, N] - # rs: should be the same, no? - assert center0_d[0,0] == centers_d[0,-1] and center0_d[1,0] == centers_d[1,-1] + + contour0_d = contours_only_text_parent_d[-1] + center0_d = centers_d[:, -1:] # [2, 1] + last5_centers_d = centers_d[:, -5:] dists_d = np.linalg.norm(center0 - last5_centers_d, axis=0) ind_largest = len(contours_only_text_parent_d) - last5_centers_d.shape[1] + np.argmin(dists_d) From 0f33c21eb3a9cbe87f7221dd3481203de415794d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Sun, 5 Oct 2025 02:45:01 +0200 Subject: [PATCH 326/374] matching deskewed text region contours with predicted: improve - when matching undeskewed and new contours, do not just pick the closest centers, respectively, but also of similar size (by making the contour area the 3rd dimension of the vector norm in the distance calculation) --- src/eynollah/eynollah.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 46437f0..e474916 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4610,7 +4610,11 @@ class Eynollah: for i in range(len(contours_only_text_parent)): p = np.dot(M_22, centers[:, i:i+1]) # [2, 1] p -= offset - dists = np.linalg.norm(p - centers_d, axis=0) + # add dimension for area + #dists = np.linalg.norm(p - centers_d, axis=0) + diffs = (np.append(p, [[areas_cnt_text_parent[i]]], axis=0) - + np.append(centers_d, areas_cnt_text_d[np.newaxis], axis=0)) + dists = np.linalg.norm(diffs, axis=0) contours_only_text_parent_d_ordered.append( contours_only_text_parent_d[np.argmin(dists)]) # cv2.fillPoly(img2, pts=[contours_only_text_parent_d[np.argmin(dists)]], color=i + 1) From 0e00d7868be55d3fb94b52fffc6ed96bf9387067 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 12:55:10 +0200 Subject: [PATCH 327/374] matching deskewed text region contours with predicted: improve - apply same min-area filter to deskewed contours as to original ones --- src/eynollah/eynollah.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index e474916..e5ad5ae 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4568,7 +4568,6 @@ class Eynollah: centers = np.stack(find_center_of_contours(contours_only_text_parent)) # [2, N] - contour0 = contours_only_text_parent[-1] center0 = centers[:, -1:] # [2, 1] if np.abs(slope_deskew) >= SLOPE_THRESHOLD: @@ -4578,6 +4577,9 @@ class Eynollah: areas_cnt_text_d = np.array([cv2.contourArea(c) for c in contours_only_text_parent_d]) areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) + contours_only_text_parent_d = np.array(contours_only_text_parent_d)[areas_cnt_text_d > MIN_AREA_REGION] + areas_cnt_text_d = areas_cnt_text_d[areas_cnt_text_d > MIN_AREA_REGION] + if len(contours_only_text_parent_d): index_con_parents_d = np.argsort(areas_cnt_text_d) contours_only_text_parent_d = np.array(contours_only_text_parent_d)[index_con_parents_d] @@ -4585,9 +4587,10 @@ class Eynollah: centers_d = np.stack(find_center_of_contours(contours_only_text_parent_d)) # [2, N] - contour0_d = contours_only_text_parent_d[-1] center0_d = centers_d[:, -1:] # [2, 1] + # find the largest among the largest 5 deskewed contours + # that is also closest to the largest original contour last5_centers_d = centers_d[:, -5:] dists_d = np.linalg.norm(center0 - last5_centers_d, axis=0) ind_largest = len(contours_only_text_parent_d) - last5_centers_d.shape[1] + np.argmin(dists_d) @@ -4762,14 +4765,7 @@ class Eynollah: if np.abs(slope_deskew) >= SLOPE_THRESHOLD: contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( contours_only_text_parent_d_ordered, index_by_text_par_con) - #try: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=np.int32)[index_by_text_par_con]) - #except: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=object)[index_by_text_par_con]) else: - #takes long timee contours_only_text_parent_d_ordered = None if self.light_version: fun = check_any_text_region_in_model_one_is_main_or_header_light @@ -4949,12 +4945,6 @@ class Eynollah: else: contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( contours_only_text_parent_d_ordered, index_by_text_par_con) - #try: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=object)[index_by_text_par_con]) - #except: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=np.int32)[index_by_text_par_con]) order_text_new, id_of_texts_tot = self.do_order_of_regions( contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) From 155b8f68b8a7754de11e002e0df2bfc7292899d8 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 12:58:24 +0200 Subject: [PATCH 328/374] matching deskewed text region contours with predicted: improve - avoid duplicate and missing mappings by using a different approach: instead of just minimising the center distance for the N contours that we expect, 1. get all N:M distances 2. iterate over them from small to large 3. continue adding correspondences until both every original contour and every deskewed contour have at least one match 4. where one original matches multiple deskewed contours, join the latter polygons to map as single contour 5. where one deskewed contour matches multiple originals, split the former by intersecting with each of the latter (after bringing them into the same coordinate space), so ultimately only the respective match gets assigned --- src/eynollah/eynollah.py | 94 ++++++++++++++++++++++++++++------- src/eynollah/utils/contour.py | 15 ++++++ 2 files changed, 90 insertions(+), 19 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index e5ad5ae..5e32929 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -33,6 +33,7 @@ from concurrent.futures import ProcessPoolExecutor import xml.etree.ElementTree as ET import cv2 import numpy as np +import shapely.affinity from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d from numba import cuda @@ -83,6 +84,10 @@ from .utils.contour import ( return_parent_contours, dilate_textregion_contours, dilate_textline_contours, + polygon2contour, + contour2polygon, + join_polygons, + make_intersection, ) from .utils.rotate import ( rotate_image, @@ -4556,8 +4561,9 @@ class Eynollah: contours_only_text, hir_on_text = return_contours_of_image(text_only) contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) if len(contours_only_text_parent) > 0: + areas_tot_text = np.prod(text_only.shape) areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) - areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) + areas_cnt_text = areas_cnt_text / float(areas_tot_text) #self.logger.info('areas_cnt_text %s', areas_cnt_text) contours_only_text_parent = np.array(contours_only_text_parent)[areas_cnt_text > MIN_AREA_REGION] areas_cnt_text_parent = areas_cnt_text[areas_cnt_text > MIN_AREA_REGION] @@ -4574,8 +4580,9 @@ class Eynollah: contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) contours_only_text_parent_d = return_parent_contours(contours_only_text_d, hir_on_text_d) + areas_tot_text_d = np.prod(text_only_d.shape) areas_cnt_text_d = np.array([cv2.contourArea(c) for c in contours_only_text_parent_d]) - areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) + areas_cnt_text_d = areas_cnt_text_d / float(areas_tot_text_d) contours_only_text_parent_d = np.array(contours_only_text_parent_d)[areas_cnt_text_d > MIN_AREA_REGION] areas_cnt_text_d = areas_cnt_text_d[areas_cnt_text_d > MIN_AREA_REGION] @@ -4587,7 +4594,7 @@ class Eynollah: centers_d = np.stack(find_center_of_contours(contours_only_text_parent_d)) # [2, N] - center0_d = centers_d[:, -1:] # [2, 1] + center0_d = centers_d[:, -1:].copy() # [2, 1] # find the largest among the largest 5 deskewed contours # that is also closest to the largest original contour @@ -4605,26 +4612,75 @@ class Eynollah: center = (w // 2.0, h // 2.0) M = cv2.getRotationMatrix2D(center, slope_deskew, 1.0) M_22 = np.array(M)[:2, :2] - p0 = np.dot(M_22, center0) # [2, 1] - offset = p0 - center0_d # [2, 1] + center0 = np.dot(M_22, center0) # [2, 1] + offset = center0 - center0_d # [2, 1] - # img2 = np.zeros(text_only_d.shape[:2], dtype=np.uint8) - contours_only_text_parent_d_ordered = [] + centers = np.dot(M_22, centers) - offset # [2,N] + # add dimension for area (so only contours of similar size will be considered close) + centers = np.append(centers, areas_cnt_text_parent[np.newaxis], axis=0) + centers_d = np.append(centers_d, areas_cnt_text_d[np.newaxis], axis=0) + + dists = np.zeros((len(contours_only_text_parent), len(contours_only_text_parent_d))) for i in range(len(contours_only_text_parent)): - p = np.dot(M_22, centers[:, i:i+1]) # [2, 1] - p -= offset - # add dimension for area - #dists = np.linalg.norm(p - centers_d, axis=0) - diffs = (np.append(p, [[areas_cnt_text_parent[i]]], axis=0) - - np.append(centers_d, areas_cnt_text_d[np.newaxis], axis=0)) - dists = np.linalg.norm(diffs, axis=0) - contours_only_text_parent_d_ordered.append( - contours_only_text_parent_d[np.argmin(dists)]) - # cv2.fillPoly(img2, pts=[contours_only_text_parent_d[np.argmin(dists)]], color=i + 1) + dists[i] = np.linalg.norm(centers[:, i:i + 1] - centers_d, axis=0) + corresp = np.zeros(dists.shape, dtype=bool) + # keep searching next-closest until at least one correspondence on each side + while not np.all(corresp.sum(axis=1)) and not np.all(corresp.sum(axis=0)): + idx = np.nanargmin(dists) + i, j = np.unravel_index(idx, dists.shape) + dists[i, j] = np.nan + corresp[i, j] = True + #print("original/deskewed adjacency", corresp.nonzero()) + contours_only_text_parent_d_ordered = np.zeros_like(contours_only_text_parent) + contours_only_text_parent_d_ordered = contours_only_text_parent_d[np.argmax(corresp, axis=1)] + # img1 = np.zeros(text_only_d.shape[:2], dtype=np.uint8) + # for i in range(len(contours_only_text_parent)): + # cv2.fillPoly(img1, pts=[contours_only_text_parent_d_ordered[i]], color=i + 1) + # plt.subplot(2, 2, 1, title="direct corresp contours") + # plt.imshow(img1) + # img2 = np.zeros(text_only_d.shape[:2], dtype=np.uint8) + # join deskewed regions mapping to single original ones + for i in range(len(contours_only_text_parent)): + if np.count_nonzero(corresp[i]) > 1: + indices = np.flatnonzero(corresp[i]) + #print("joining", indices) + polygons_d = [contour2polygon(contour) + for contour in contours_only_text_parent_d[indices]] + contour_d = polygon2contour(join_polygons(polygons_d)) + contours_only_text_parent_d_ordered[i] = contour_d + # cv2.fillPoly(img2, pts=[contour_d], color=i + 1) + # plt.subplot(2, 2, 3, title="joined contours") # plt.imshow(img2) + # img3 = np.zeros(text_only_d.shape[:2], dtype=np.uint8) + # split deskewed regions mapping to multiple original ones + def deskew(polygon): + polygon = shapely.affinity.rotate(polygon, -slope_deskew, origin=center) + polygon = shapely.affinity.translate(polygon, *offset.squeeze()) + return polygon + for j in range(len(contours_only_text_parent_d)): + if np.count_nonzero(corresp[:, j]) > 1: + indices = np.flatnonzero(corresp[:, j]) + #print("splitting along", indices) + polygons = [deskew(contour2polygon(contour)) + for contour in contours_only_text_parent[indices]] + polygon_d = contour2polygon(contours_only_text_parent_d[j]) + polygons_d = [make_intersection(polygon_d, polygon) + for polygon in polygons] + # ignore where there is no actual overlap + indices = indices[np.flatnonzero(polygons_d)] + contours_d = [polygon2contour(polygon_d) + for polygon_d in polygons_d + if polygon_d] + contours_only_text_parent_d_ordered[indices] = contours_d + # cv2.fillPoly(img3, pts=contours_d, color=j + 1) + # plt.subplot(2, 2, 4, title="split contours") + # plt.imshow(img3) + # img4 = np.zeros(text_only_d.shape[:2], dtype=np.uint8) + # for i in range(len(contours_only_text_parent)): + # cv2.fillPoly(img4, pts=[contours_only_text_parent_d_ordered[i]], color=i + 1) + # plt.subplot(2, 2, 2, title="result contours") + # plt.imshow(img4) # plt.show() - # rs: what about the remaining contours_only_text_parent_d? - # rs: what about duplicates? else: contours_only_text_parent_d_ordered = [] contours_only_text_parent_d = [] diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 041cbf6..8431bbe 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -335,6 +335,21 @@ def polygon2contour(polygon: Polygon) -> np.ndarray: polygon = np.array(polygon.exterior.coords[:-1], dtype=int) return np.maximum(0, polygon).astype(np.uint)[:, np.newaxis] +def make_intersection(poly1, poly2): + interp = poly1.intersection(poly2) + # post-process + if interp.is_empty or interp.area == 0.0: + return None + if interp.geom_type == 'GeometryCollection': + # heterogeneous result: filter zero-area shapes (LineString, Point) + interp = unary_union([geom for geom in interp.geoms if geom.area > 0]) + if interp.geom_type == 'MultiPolygon': + # homogeneous result: construct convex hull to connect + interp = join_polygons(interp.geoms) + assert interp.geom_type == 'Polygon', interp.wkt + interp = make_valid(interp) + return interp + def make_valid(polygon: Polygon) -> Polygon: """Ensures shapely.geometry.Polygon object is valid by repeated rearrangement/simplification/enlargement.""" def isint(x): From fe603188f4f7f9d545b44085cdc45195f98f0546 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 13:11:03 +0200 Subject: [PATCH 329/374] avoid unnecessary 3-channel conversions --- src/eynollah/eynollah.py | 52 ++++----- src/eynollah/utils/__init__.py | 156 +++++++++++---------------- src/eynollah/utils/contour.py | 74 +++++-------- src/eynollah/utils/separate_lines.py | 53 ++++----- 4 files changed, 132 insertions(+), 203 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 5e32929..834ecf3 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -712,7 +712,7 @@ class Eynollah: if self.input_binary: img = self.imread() prediction_bin = self.do_prediction(True, img, self.model_bin, n_batch_inference=5) - prediction_bin = 255 * (prediction_bin[:,:,0]==0) + prediction_bin = 255 * (prediction_bin[:,:,0] == 0) prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2).astype(np.uint8) img= np.copy(prediction_bin) img_bin = prediction_bin @@ -2064,9 +2064,7 @@ class Eynollah: boxes_sub_new = [] poly_sub = [] for mv in range(len(boxes_per_process)): - crop_img, _ = crop_image_inside_box(boxes_per_process[mv], - np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) - crop_img = crop_img[:, :, 0] + crop_img, _ = crop_image_inside_box(boxes_per_process[mv], textline_mask_tot) crop_img = cv2.erode(crop_img, KERNEL, iterations=2) try: textline_con, hierarchy = return_contours_of_image(crop_img) @@ -2638,10 +2636,8 @@ class Eynollah: layout_org[:,:,0][layout_org[:,:,0]==pixel_table] = 0 layout = (layout[:,:,0]==pixel_table)*1 - layout =np.repeat(layout[:, :, np.newaxis], 3, axis=2) layout = layout.astype(np.uint8) - imgray = cv2.cvtColor(layout, cv2.COLOR_BGR2GRAY ) - _, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(layout, 0, 255, 0) contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cnt_size = np.array([cv2.contourArea(contours[j]) @@ -2652,8 +2648,8 @@ class Eynollah: x, y, w, h = cv2.boundingRect(contours[i]) iou = cnt_size[i] /float(w*h) *100 if iou<80: - layout_contour = np.zeros((layout_org.shape[0], layout_org.shape[1])) - layout_contour= cv2.fillPoly(layout_contour,pts=[contours[i]] ,color=(1,1,1)) + layout_contour = np.zeros(layout_org.shape[:2]) + layout_contour = cv2.fillPoly(layout_contour, pts=[contours[i]] ,color=1) layout_contour_sum = layout_contour.sum(axis=0) layout_contour_sum_diff = np.diff(layout_contour_sum) @@ -2669,20 +2665,17 @@ class Eynollah: layout_contour=cv2.erode(layout_contour[:,:], KERNEL, iterations=5) layout_contour=cv2.dilate(layout_contour[:,:], KERNEL, iterations=5) - layout_contour =np.repeat(layout_contour[:, :, np.newaxis], 3, axis=2) layout_contour = layout_contour.astype(np.uint8) - - imgray = cv2.cvtColor(layout_contour, cv2.COLOR_BGR2GRAY ) - _, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(layout_contour, 0, 255, 0) contours_sep, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) for ji in range(len(contours_sep) ): contours_new.append(contours_sep[ji]) if num_col_classifier>=2: - only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1])) - only_recent_contour_image= cv2.fillPoly(only_recent_contour_image, - pts=[contours_sep[ji]], color=(1,1,1)) + only_recent_contour_image = np.zeros(layout.shape[:2]) + only_recent_contour_image = cv2.fillPoly(only_recent_contour_image, + pts=[contours_sep[ji]], color=1) table_pixels_masked_from_early_pre = only_recent_contour_image * table_prediction_early iou_in = 100. * table_pixels_masked_from_early_pre.sum() / only_recent_contour_image.sum() #print(iou_in,'iou_in_in1') @@ -3210,13 +3203,11 @@ class Eynollah: pixel_lines = 3 if np.abs(slope_deskew) < SLOPE_THRESHOLD: _, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) + text_regions_p, num_col_classifier, self.tables, pixel_lines) if np.abs(slope_deskew) >= SLOPE_THRESHOLD: _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) + text_regions_p_1_n, num_col_classifier, self.tables, pixel_lines) #print(time.time()-t_0_box,'time box in 2') self.logger.info("num_col_classifier: %s", num_col_classifier) @@ -3392,13 +3383,11 @@ class Eynollah: pixel_lines=3 if np.abs(slope_deskew) < SLOPE_THRESHOLD: num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) + text_regions_p, num_col_classifier, self.tables, pixel_lines) if np.abs(slope_deskew) >= SLOPE_THRESHOLD: num_col_d, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) + text_regions_p_1_n, num_col_classifier, self.tables, pixel_lines) if num_col_classifier>=3: if np.abs(slope_deskew) < SLOPE_THRESHOLD: @@ -3498,7 +3487,7 @@ class Eynollah: #text_regions_p[:,:][regions_fully[:,:,0]==6]=6 ##regions_fully_only_drop = put_drop_out_from_only_drop_model(regions_fully_only_drop, text_regions_p) - ##regions_fully[:, :, 0][regions_fully_only_drop[:, :, 0] == 4] = 4 + ##regions_fully[:, :, 0][regions_fully_only_drop[:, :] == 4] = 4 drop_capital_label_in_full_layout_model = 3 drops = (regions_fully[:,:,0]==drop_capital_label_in_full_layout_model)*1 @@ -4715,7 +4704,6 @@ class Eynollah: return pcgts - #print("text region early 3 in %.1fs", time.time() - t0) if self.light_version: contours_only_text_parent = dilate_textregion_contours(contours_only_text_parent) @@ -4851,21 +4839,17 @@ class Eynollah: if not self.headers_off: if np.abs(slope_deskew) < SLOPE_THRESHOLD: num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, label_seps, contours_only_text_parent_h) + text_regions_p, num_col_classifier, self.tables, label_seps, contours_only_text_parent_h) else: _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, label_seps, contours_only_text_parent_h_d_ordered) + text_regions_p_1_n, num_col_classifier, self.tables, label_seps, contours_only_text_parent_h_d_ordered) elif self.headers_off: if np.abs(slope_deskew) < SLOPE_THRESHOLD: num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, label_seps) + text_regions_p, num_col_classifier, self.tables, label_seps) else: _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, label_seps) + text_regions_p_1_n, num_col_classifier, self.tables, label_seps) if num_col_classifier >= 3: if np.abs(slope_deskew) < SLOPE_THRESHOLD: diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index 6e5afd4..ebf78fe 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -796,7 +796,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): return len(peaks_fin_true), peaks_fin_true def find_num_col_by_vertical_lines(regions_without_separators, multiplier=3.8): - regions_without_separators_0 = regions_without_separators[:, :, 0].sum(axis=0) + regions_without_separators_0 = regions_without_separators.sum(axis=0) ##plt.plot(regions_without_separators_0) ##plt.show() @@ -823,7 +823,10 @@ def return_regions_without_separators(regions_pre): return regions_without_separators def put_drop_out_from_only_drop_model(layout_no_patch, layout1): - drop_only = (layout_no_patch[:, :, 0] == 4) * 1 + if layout_no_patch.ndim == 3: + layout_no_patch = layout_no_patch[:, :, 0] + + drop_only = (layout_no_patch[:, :] == 4) * 1 contours_drop, hir_on_drop = return_contours_of_image(drop_only) contours_drop_parent = return_parent_contours(contours_drop, hir_on_drop) @@ -849,9 +852,8 @@ def put_drop_out_from_only_drop_model(layout_no_patch, layout1): (map_of_drop_contour_bb == 5).sum()) >= 15: contours_drop_parent_final.append(contours_drop_parent[jj]) - layout_no_patch[:, :, 0][layout_no_patch[:, :, 0] == 4] = 0 - - layout_no_patch = cv2.fillPoly(layout_no_patch, pts=contours_drop_parent_final, color=(4, 4, 4)) + layout_no_patch[:, :][layout_no_patch[:, :] == 4] = 0 + layout_no_patch = cv2.fillPoly(layout_no_patch, pts=contours_drop_parent_final, color=4) return layout_no_patch @@ -925,17 +927,16 @@ def check_any_text_region_in_model_one_is_main_or_header( contours_only_text_parent_main_d=[] contours_only_text_parent_head_d=[] - for ii in range(len(contours_only_text_parent)): - con=contours_only_text_parent[ii] - img=np.zeros((regions_model_1.shape[0],regions_model_1.shape[1],3)) - img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) + for ii, con in enumerate(contours_only_text_parent): + img = np.zeros(regions_model_1.shape[:2]) + img = cv2.fillPoly(img, pts=[con], color=255) - all_pixels=((img[:,:,0]==255)*1).sum() - pixels_header=( ( (img[:,:,0]==255) & (regions_model_full[:,:,0]==2) )*1 ).sum() + all_pixels=((img == 255)*1).sum() + pixels_header=( ( (img == 255) & (regions_model_full[:,:,0]==2) )*1 ).sum() pixels_main=all_pixels-pixels_header if (pixels_header>=pixels_main) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ]=2 contours_only_text_parent_head.append(con) if contours_only_text_parent_d_ordered is not None: contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) @@ -944,7 +945,7 @@ def check_any_text_region_in_model_one_is_main_or_header( all_found_textline_polygons_head.append(all_found_textline_polygons[ii]) conf_contours_head.append(None) else: - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ]=1 contours_only_text_parent_main.append(con) conf_contours_main.append(conf_contours[ii]) if contours_only_text_parent_d_ordered is not None: @@ -1015,11 +1016,11 @@ def check_any_text_region_in_model_one_is_main_or_header_light( contours_only_text_parent_head_d=[] for ii, con in enumerate(contours_only_text_parent_z): - img=np.zeros((regions_model_1.shape[0], regions_model_1.shape[1], 3)) - img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) + img = np.zeros(regions_model_1.shape[:2]) + img = cv2.fillPoly(img, pts=[con], color=255) - all_pixels = (img[:,:,0]==255).sum() - pixels_header=((img[:,:,0]==255) & + all_pixels = (img == 255).sum() + pixels_header=((img == 255) & (regions_model_full[:,:,0]==2)).sum() pixels_main = all_pixels - pixels_header @@ -1029,7 +1030,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light( ( pixels_header / float(pixels_main) >= 0.3 and length_con[ii] / float(height_con[ii]) >=3 )): - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ] = 2 + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ] = 2 contours_only_text_parent_head.append(contours_only_text_parent[ii]) conf_contours_head.append(None) # why not conf_contours[ii], too? if contours_only_text_parent_d_ordered is not None: @@ -1039,7 +1040,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light( all_found_textline_polygons_head.append(all_found_textline_polygons[ii]) else: - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ] = 1 + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ] = 1 contours_only_text_parent_main.append(contours_only_text_parent[ii]) conf_contours_main.append(conf_contours[ii]) if contours_only_text_parent_d_ordered is not None: @@ -1119,11 +1120,11 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) textlines_big.append(textlines_tot[i]) textlines_big_org_form.append(textlines_tot_org_form[i]) - img_textline_s = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) - img_textline_s = cv2.fillPoly(img_textline_s, pts=textlines_small, color=(1, 1, 1)) + img_textline_s = np.zeros(textline_iamge.shape[:2]) + img_textline_s = cv2.fillPoly(img_textline_s, pts=textlines_small, color=1) - img_textline_b = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) - img_textline_b = cv2.fillPoly(img_textline_b, pts=textlines_big, color=(1, 1, 1)) + img_textline_b = np.zeros(textline_iamge.shape[:2]) + img_textline_b = cv2.fillPoly(img_textline_b, pts=textlines_big, color=1) sum_small_big_all = img_textline_s + img_textline_b sum_small_big_all2 = (sum_small_big_all[:, :] == 2) * 1 @@ -1135,11 +1136,11 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) # print(len(textlines_small),'small') intersections = [] for z2 in range(len(textlines_big)): - img_text = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) - img_text = cv2.fillPoly(img_text, pts=[textlines_small[z1]], color=(1, 1, 1)) + img_text = np.zeros(textline_iamge.shape[:2]) + img_text = cv2.fillPoly(img_text, pts=[textlines_small[z1]], color=1) - img_text2 = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) - img_text2 = cv2.fillPoly(img_text2, pts=[textlines_big[z2]], color=(1, 1, 1)) + img_text2 = np.zeros(textline_iamge.shape[:2]) + img_text2 = cv2.fillPoly(img_text2, pts=[textlines_big[z2]], color=1) sum_small_big = img_text2 + img_text sum_small_big_2 = (sum_small_big[:, :] == 2) * 1 @@ -1165,19 +1166,17 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) index_small_textlines = list(np.where(np.array(dis_small_from_bigs_tot) == z)[0]) # print(z,index_small_textlines) - img_text2 = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1], 3)) - img_text2 = cv2.fillPoly(img_text2, pts=[textlines_big[z]], color=(255, 255, 255)) + img_text2 = np.zeros(textline_iamge.shape[:2], dtype=np.uint8) + img_text2 = cv2.fillPoly(img_text2, pts=[textlines_big[z]], color=255) textlines_big_with_change.append(z) for k in index_small_textlines: - img_text2 = cv2.fillPoly(img_text2, pts=[textlines_small[k]], color=(255, 255, 255)) + img_text2 = cv2.fillPoly(img_text2, pts=[textlines_small[k]], color=255) textlines_small_with_change.append(k) - img_text2 = img_text2.astype(np.uint8) - imgray = cv2.cvtColor(img_text2, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - cont, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + _, thresh = cv2.threshold(img_text2, 0, 255, 0) + cont, _ = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # print(cont[0],type(cont)) textlines_big_with_change_con.append(cont) @@ -1189,9 +1188,8 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) # print(textlines_big_with_change,'textlines_big_with_change') # print(textlines_small_with_change,'textlines_small_with_change') # print(textlines_big) - textlines_con_changed.append(textlines_big_org_form) - else: - textlines_con_changed.append(textlines_big_org_form) + + textlines_con_changed.append(textlines_big_org_form) return textlines_con_changed def order_of_regions(textline_mask, contours_main, contours_head, y_ref): @@ -1262,29 +1260,22 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( img_p_in_ver, img_in_hor,num_col_classifier): #img_p_in_ver = cv2.erode(img_p_in_ver, self.kernel, iterations=2) - img_p_in_ver=img_p_in_ver.astype(np.uint8) - img_p_in_ver=np.repeat(img_p_in_ver[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(img_p_in_ver, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_lines_ver,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + _, thresh = cv2.threshold(img_p_in_ver, 0, 255, 0) + contours_lines_ver, _ = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) slope_lines_ver, _, x_min_main_ver, _, _, _, y_min_main_ver, y_max_main_ver, cx_main_ver = \ find_features_of_lines(contours_lines_ver) for i in range(len(x_min_main_ver)): img_p_in_ver[int(y_min_main_ver[i]): int(y_min_main_ver[i])+30, int(cx_main_ver[i])-25: - int(cx_main_ver[i])+25, 0] = 0 + int(cx_main_ver[i])+25] = 0 img_p_in_ver[int(y_max_main_ver[i])-30: int(y_max_main_ver[i]), int(cx_main_ver[i])-25: - int(cx_main_ver[i])+25, 0] = 0 + int(cx_main_ver[i])+25] = 0 - img_in_hor=img_in_hor.astype(np.uint8) - img_in_hor=np.repeat(img_in_hor[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(img_in_hor, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_lines_hor,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + _, thresh = cv2.threshold(img_in_hor, 0, 255, 0) + contours_lines_hor, _ = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) slope_lines_hor, dist_x_hor, x_min_main_hor, x_max_main_hor, cy_main_hor, _, _, _, _ = \ find_features_of_lines(contours_lines_hor) @@ -1340,22 +1331,19 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( img_p_in=img_in_hor special_separators=[] - img_p_in_ver[:,:,0][img_p_in_ver[:,:,0]==255]=1 - sep_ver_hor=img_p_in+img_p_in_ver - sep_ver_hor_cross=(sep_ver_hor[:,:,0]==2)*1 - sep_ver_hor_cross=np.repeat(sep_ver_hor_cross[:, :, np.newaxis], 3, axis=2) - sep_ver_hor_cross=sep_ver_hor_cross.astype(np.uint8) - imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_cross,_=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - cx_cross, cy_cross = find_center_of_contours(contours_cross) - for ii in range(len(cx_cross)): - img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])+5:int(cx_cross[ii])+40,0]=0 - img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])-40:int(cx_cross[ii])-4,0]=0 + img_p_in_ver[img_p_in_ver == 255] = 1 + sep_ver_hor = img_p_in + img_p_in_ver + sep_ver_hor_cross = (sep_ver_hor == 2) * 1 + _, thresh = cv2.threshold(sep_ver_hor_cross.astype(np.uint8), 0, 255, 0) + contours_cross, _ = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + center_cross = np.array(find_center_of_contours(contours_cross), dtype=int) + for cx, cy in center_cross.T: + img_p_in[cy - 30: cy + 30, cx + 5: cx + 40] = 0 + img_p_in[cy - 30: cy + 30, cx - 40: cx - 4] = 0 else: img_p_in=np.copy(img_in_hor) special_separators=[] - return img_p_in[:,:,0], special_separators + return img_p_in, special_separators def return_points_with_boundies(peaks_neg_fin, first_point, last_point): peaks_neg_tot = [] @@ -1365,11 +1353,11 @@ def return_points_with_boundies(peaks_neg_fin, first_point, last_point): peaks_neg_tot.append(last_point) return peaks_neg_tot -def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, pixel_lines, contours_h=None): +def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, label_lines, contours_h=None): t_ins_c0 = time.time() - separators_closeup=( (region_pre_p[:,:,:]==pixel_lines))*1 - separators_closeup[0:110,:,:]=0 - separators_closeup[separators_closeup.shape[0]-150:,:,:]=0 + separators_closeup=( (region_pre_p[:,:]==label_lines))*1 + separators_closeup[0:110,:]=0 + separators_closeup[separators_closeup.shape[0]-150:,:]=0 kernel = np.ones((5,5),np.uint8) separators_closeup=separators_closeup.astype(np.uint8) @@ -1381,15 +1369,11 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, separators_closeup_n=separators_closeup_n.astype(np.uint8) separators_closeup_n_binary=np.zeros(( separators_closeup_n.shape[0],separators_closeup_n.shape[1]) ) - separators_closeup_n_binary[:,:]=separators_closeup_n[:,:,0] + separators_closeup_n_binary[:,:]=separators_closeup_n[:,:] separators_closeup_n_binary[:,:][separators_closeup_n_binary[:,:]!=0]=1 - gray_early=np.repeat(separators_closeup_n_binary[:, :, np.newaxis], 3, axis=2) - gray_early=gray_early.astype(np.uint8) - imgray_e = cv2.cvtColor(gray_early, cv2.COLOR_BGR2GRAY) - ret_e, thresh_e = cv2.threshold(imgray_e, 0, 255, 0) - - contours_line_e,hierarchy_e=cv2.findContours(thresh_e,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + _, thresh_e = cv2.threshold(separators_closeup_n_binary, 0, 255, 0) + contours_line_e, _ = cv2.findContours(thresh_e.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) _, dist_xe, _, _, _, _, y_min_main, y_max_main, _ = \ find_features_of_lines(contours_line_e) dist_ye = y_max_main - y_min_main @@ -1399,10 +1383,8 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, cnts_hor_e=[] for ce in args_hor_e: cnts_hor_e.append(contours_line_e[ce]) - figs_e=np.zeros(thresh_e.shape) - figs_e=cv2.fillPoly(figs_e,pts=cnts_hor_e,color=(1,1,1)) - separators_closeup_n_binary=cv2.fillPoly(separators_closeup_n_binary, pts=cnts_hor_e, color=(0,0,0)) + separators_closeup_n_binary=cv2.fillPoly(separators_closeup_n_binary, pts=cnts_hor_e, color=0) gray = cv2.bitwise_not(separators_closeup_n_binary) gray=gray.astype(np.uint8) @@ -1422,7 +1404,7 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, kernel = np.ones((5,5),np.uint8) horizontal = cv2.dilate(horizontal,kernel,iterations = 2) horizontal = cv2.erode(horizontal,kernel,iterations = 2) - horizontal = cv2.fillPoly(horizontal, pts=cnts_hor_e, color=(255,255,255)) + horizontal = cv2.fillPoly(horizontal, pts=cnts_hor_e, color=255) rows = vertical.shape[0] verticalsize = rows // 30 @@ -1440,13 +1422,8 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, separators_closeup_new[:,:][vertical[:,:]!=0]=1 separators_closeup_new[:,:][horizontal[:,:]!=0]=1 - vertical=np.repeat(vertical[:, :, np.newaxis], 3, axis=2) - vertical=vertical.astype(np.uint8) - - imgray = cv2.cvtColor(vertical, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_line_vers,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + _, thresh = cv2.threshold(vertical, 0, 255, 0) + contours_line_vers, _ = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = \ find_features_of_lines(contours_line_vers) @@ -1461,11 +1438,8 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, dist_y_ver=y_max_main_ver-y_min_main_ver len_y=separators_closeup.shape[0]/3.0 - horizontal=np.repeat(horizontal[:, :, np.newaxis], 3, axis=2) - horizontal=horizontal.astype(np.uint8) - imgray = cv2.cvtColor(horizontal, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_line_hors,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + _, thresh = cv2.threshold(horizontal, 0, 255, 0) + contours_line_hors, _ = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = \ find_features_of_lines(contours_line_hors) @@ -1558,7 +1532,7 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, peaks_neg_fin_fin=[] for itiles in args_big_parts: regions_without_separators_tile=regions_without_separators[int(splitter_y_new[itiles]): - int(splitter_y_new[itiles+1]),:,0] + int(splitter_y_new[itiles+1]),:] try: num_col, peaks_neg_fin = find_num_col(regions_without_separators_tile, num_col_classifier, tables, multiplier=7.0) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 8431bbe..22a6f50 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -119,14 +119,11 @@ def return_parent_contours(contours, hierarchy): def return_contours_of_interested_region(region_pre_p, label, min_area=0.0002): # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: + if region_pre_p.ndim == 3: cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: cnts_images = (region_pre_p[:, :] == label) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours_imgs = return_parent_contours(contours_imgs, hierarchy) @@ -135,13 +132,11 @@ def return_contours_of_interested_region(region_pre_p, label, min_area=0.0002): return contours_imgs def do_work_of_contours_in_image(contour, index_r_con, img, slope_first): - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[contour], color=(1, 1, 1)) + img_copy = np.zeros(img.shape[:2], dtype=np.uint8) + img_copy = cv2.fillPoly(img_copy, pts=[contour], color=1) img_copy = rotation_image_new(img_copy, -slope_first) - img_copy = img_copy.astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(img_copy, 0, 255, 0) cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) @@ -164,8 +159,8 @@ def get_textregion_contours_in_org_image(cnts, img, slope_first): cnts_org = [] # print(cnts,'cnts') for i in range(len(cnts)): - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[cnts[i]], color=(1, 1, 1)) + img_copy = np.zeros(img.shape[:2], dtype=np.uint8) + img_copy = cv2.fillPoly(img_copy, pts=[cnts[i]], color=1) # plt.imshow(img_copy) # plt.show() @@ -176,9 +171,7 @@ def get_textregion_contours_in_org_image(cnts, img, slope_first): # plt.imshow(img_copy) # plt.show() - img_copy = img_copy.astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(img_copy, 0, 255, 0) cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) @@ -195,12 +188,11 @@ def get_textregion_contours_in_org_image_light_old(cnts, img, slope_first): interpolation=cv2.INTER_NEAREST) cnts_org = [] for cnt in cnts: - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[(cnt / zoom).astype(int)], color=(1, 1, 1)) + img_copy = np.zeros(img.shape[:2], dtype=np.uint8) + img_copy = cv2.fillPoly(img_copy, pts=[cnt // zoom], color=1) img_copy = rotation_image_new(img_copy, -slope_first).astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(img_copy, 0, 255, 0) cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) @@ -210,14 +202,13 @@ def get_textregion_contours_in_org_image_light_old(cnts, img, slope_first): return cnts_org def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first, confidence_matrix): - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[contour_par], color=(1, 1, 1)) - confidence_matrix_mapped_with_contour = confidence_matrix * img_copy[:,:,0] - confidence_contour = np.sum(confidence_matrix_mapped_with_contour) / float(np.sum(img_copy[:,:,0])) + img_copy = np.zeros(img.shape[:2], dtype=np.uint8) + img_copy = cv2.fillPoly(img_copy, pts=[contour_par], color=1) + confidence_matrix_mapped_with_contour = confidence_matrix * img_copy + confidence_contour = np.sum(confidence_matrix_mapped_with_contour) / float(np.sum(img_copy)) img_copy = rotation_image_new(img_copy, -slope_first).astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(img_copy, 0, 255, 0) cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) if len(cont_int)==0: @@ -245,14 +236,11 @@ def get_textregion_contours_in_org_image_light(cnts, img, confidence_matrix): def return_contours_of_interested_textline(region_pre_p, label): # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: + if region_pre_p.ndim == 3: cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: cnts_images = (region_pre_p[:, :] == label) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours_imgs = return_parent_contours(contours_imgs, hierarchy) @@ -262,25 +250,22 @@ def return_contours_of_interested_textline(region_pre_p, label): def return_contours_of_image(image): if len(image.shape) == 2: - image = np.repeat(image[:, :, np.newaxis], 3, axis=2) image = image.astype(np.uint8) + imgray = image else: image = image.astype(np.uint8) - imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) return contours, hierarchy def return_contours_of_interested_region_by_min_size(region_pre_p, label, min_size=0.00003): # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: + if region_pre_p.ndim == 3: cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: cnts_images = (region_pre_p[:, :] == label) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours_imgs = return_parent_contours(contours_imgs, hierarchy) @@ -291,24 +276,21 @@ def return_contours_of_interested_region_by_min_size(region_pre_p, label, min_si def return_contours_of_interested_region_by_size(region_pre_p, label, min_area, max_area): # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: + if region_pre_p.ndim == 3: cnts_images = (region_pre_p[:, :, 0] == label) * 1 else: cnts_images = (region_pre_p[:, :] == label) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours_imgs = return_parent_contours(contours_imgs, hierarchy) contours_imgs = filter_contours_area_of_image_tables( thresh, contours_imgs, hierarchy, max_area=max_area, min_area=min_area) - img_ret = np.zeros((region_pre_p.shape[0], region_pre_p.shape[1], 3)) - img_ret = cv2.fillPoly(img_ret, pts=contours_imgs, color=(1, 1, 1)) + img_ret = np.zeros((region_pre_p.shape[0], region_pre_p.shape[1])) + img_ret = cv2.fillPoly(img_ret, pts=contours_imgs, color=1) - return img_ret[:, :, 0] + return img_ret def dilate_textline_contours(all_found_textline_polygons): return [[polygon2contour(contour2polygon(contour, dilate=6)) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index d41dda1..b8c7f3d 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -142,13 +142,12 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): rotation_matrix) def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): - (h, w) = img_patch.shape[:2] + h, w = img_patch.shape[:2] center = (w // 2, h // 2) M = cv2.getRotationMatrix2D(center, -thetha, 1.0) x_d = M[0, 2] y_d = M[1, 2] - thetha = thetha / 180. * np.pi - rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + rotation_matrix = M[:2, :2] contour_text_interest_copy = contour_text_interest.copy() x_cont = contour_text_interest[:, 0, 0] @@ -1302,19 +1301,16 @@ def separate_lines_new_inside_tiles(img_path, thetha): def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_ind, add_boxes_coor_into_textlines): kernel = np.ones((5, 5), np.uint8) - pixel = 255 + label = 255 min_area = 0 max_area = 1 - if len(img_patch.shape) == 3: - cnts_images = (img_patch[:, :, 0] == pixel) * 1 + if img_patch.ndim == 3: + cnts_images = (img_patch[:, :, 0] == label) * 1 else: - cnts_images = (img_patch[:, :] == pixel) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + cnts_images = (img_patch[:, :] == label) * 1 + _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) + contours_imgs, hierarchy = cv2.findContours(thresh.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours_imgs = return_parent_contours(contours_imgs, hierarchy) contours_imgs = filter_contours_area_of_image_tables(thresh, @@ -1322,14 +1318,12 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i max_area=max_area, min_area=min_area) cont_final = [] for i in range(len(contours_imgs)): - img_contour = np.zeros((cnts_images.shape[0], cnts_images.shape[1], 3)) - img_contour = cv2.fillPoly(img_contour, pts=[contours_imgs[i]], color=(255, 255, 255)) - img_contour = img_contour.astype(np.uint8) + img_contour = np.zeros(cnts_images.shape[:2], dtype=np.uint8) + img_contour = cv2.fillPoly(img_contour, pts=[contours_imgs[i]], color=255) img_contour = cv2.dilate(img_contour, kernel, iterations=4) - imgrayrot = cv2.cvtColor(img_contour, cv2.COLOR_BGR2GRAY) - _, threshrot = cv2.threshold(imgrayrot, 0, 255, 0) - contours_text_rot, _ = cv2.findContours(threshrot.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + _, threshrot = cv2.threshold(img_contour, 0, 255, 0) + contours_text_rot, _ = cv2.findContours(threshrot.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) ##contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[ ##0] @@ -1344,8 +1338,7 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i def textline_contours_postprocessing(textline_mask, slope, contour_text_interest, box_ind, add_boxes_coor_into_textlines=False): - textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 - textline_mask = textline_mask.astype(np.uint8) + textline_mask = textline_mask * 255 kernel = np.ones((5, 5), np.uint8) textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_OPEN, kernel) textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, kernel) @@ -1356,12 +1349,11 @@ def textline_contours_postprocessing(textline_mask, slope, y_help = 2 textline_mask_help = np.zeros((textline_mask.shape[0] + int(2 * y_help), - textline_mask.shape[1] + int(2 * x_help), 3)) + textline_mask.shape[1] + int(2 * x_help))) textline_mask_help[y_help : y_help + textline_mask.shape[0], - x_help : x_help + textline_mask.shape[1], :] = np.copy(textline_mask[:, :, :]) + x_help : x_help + textline_mask.shape[1]] = np.copy(textline_mask[:, :]) dst = rotate_image(textline_mask_help, slope) - dst = dst[:, :, 0] dst[dst != 0] = 1 # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: @@ -1372,21 +1364,18 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[0] contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] - img_contour = np.zeros((box_ind[3], box_ind[2], 3)) - img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=(255, 255, 255)) + img_contour = np.zeros((box_ind[3], box_ind[2])) + img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=255) img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), - img_contour.shape[1] + int(2 * x_help), 3)) + img_contour.shape[1] + int(2 * x_help))) img_contour_help[y_help : y_help + img_contour.shape[0], - x_help : x_help + img_contour.shape[1], :] = np.copy(img_contour[:, :, :]) + x_help : x_help + img_contour.shape[1]] = np.copy(img_contour[:, :]) img_contour_rot = rotate_image(img_contour_help, slope) - img_contour_rot = img_contour_rot.astype(np.uint8) - # dst_help = dst_help.astype(np.uint8) - imgrayrot = cv2.cvtColor(img_contour_rot, cv2.COLOR_BGR2GRAY) - _, threshrot = cv2.threshold(imgrayrot, 0, 255, 0) - contours_text_rot, _ = cv2.findContours(threshrot.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + _, threshrot = cv2.threshold(img_contour_rot, 0, 255, 0) + contours_text_rot, _ = cv2.findContours(threshrot.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] ind_big_con = np.argmax(len_con_text_rot) From 6e57ab3741f5532a30dd2925b423cd40871ab010 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 16:53:59 +0200 Subject: [PATCH 330/374] textline_contours_postprocessing: do not catch arbitrary exceptions --- src/eynollah/utils/separate_lines.py | 68 ++++++++++++++-------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index b8c7f3d..3bfc903 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1344,51 +1344,49 @@ def textline_contours_postprocessing(textline_mask, slope, textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, kernel) textline_mask = cv2.erode(textline_mask, kernel, iterations=2) # textline_mask = cv2.erode(textline_mask, kernel, iterations=1) - try: - x_help = 30 - y_help = 2 - textline_mask_help = np.zeros((textline_mask.shape[0] + int(2 * y_help), - textline_mask.shape[1] + int(2 * x_help))) - textline_mask_help[y_help : y_help + textline_mask.shape[0], - x_help : x_help + textline_mask.shape[1]] = np.copy(textline_mask[:, :]) + x_help = 30 + y_help = 2 - dst = rotate_image(textline_mask_help, slope) - dst[dst != 0] = 1 + textline_mask_help = np.zeros((textline_mask.shape[0] + int(2 * y_help), + textline_mask.shape[1] + int(2 * x_help))) + textline_mask_help[y_help : y_help + textline_mask.shape[0], + x_help : x_help + textline_mask.shape[1]] = np.copy(textline_mask[:, :]) - # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - # plt.imshow(dst) - # plt.show() + dst = rotate_image(textline_mask_help, slope) + dst[dst != 0] = 1 - contour_text_copy = contour_text_interest.copy() - contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[0] - contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(dst) + # plt.show() - img_contour = np.zeros((box_ind[3], box_ind[2])) - img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=255) + contour_text_copy = contour_text_interest.copy() + contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[0] + contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] - img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), - img_contour.shape[1] + int(2 * x_help))) - img_contour_help[y_help : y_help + img_contour.shape[0], - x_help : x_help + img_contour.shape[1]] = np.copy(img_contour[:, :]) + img_contour = np.zeros((box_ind[3], box_ind[2])) + img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=255) - img_contour_rot = rotate_image(img_contour_help, slope) + img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), + img_contour.shape[1] + int(2 * x_help))) + img_contour_help[y_help : y_help + img_contour.shape[0], + x_help : x_help + img_contour.shape[1]] = np.copy(img_contour[:, :]) - _, threshrot = cv2.threshold(img_contour_rot, 0, 255, 0) - contours_text_rot, _ = cv2.findContours(threshrot.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + img_contour_rot = rotate_image(img_contour_help, slope) - len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] - ind_big_con = np.argmax(len_con_text_rot) + _, threshrot = cv2.threshold(img_contour_rot, 0, 255, 0) + contours_text_rot, _ = cv2.findContours(threshrot.astype(np.uint8), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - if abs(slope) > 45: - _, contours_rotated_clean = separate_lines_vertical_cont( - textline_mask, contours_text_rot[ind_big_con], box_ind, slope, - add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) - else: - _, contours_rotated_clean = separate_lines( - dst, contours_text_rot[ind_big_con], slope, x_help, y_help) - except: - contours_rotated_clean = [] + len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] + ind_big_con = np.argmax(len_con_text_rot) + + if abs(slope) > 45: + _, contours_rotated_clean = separate_lines_vertical_cont( + textline_mask, contours_text_rot[ind_big_con], box_ind, slope, + add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) + else: + _, contours_rotated_clean = separate_lines( + dst, contours_text_rot[ind_big_con], slope, x_help, y_help) return contours_rotated_clean From 595ed02743afc3ab8359de5f6feb0ca680546599 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 17:24:50 +0200 Subject: [PATCH 331/374] run_single: simplify; allow running TrOCR in non-fl mode, too - refactor final `self.full_layout` conditional, removing copied code - allow running `self.ocr` and `self.tr` branch in both cases (non/fl) - when running TrOCR, use model / processor / device initialised during init (instead of ad-hoc loading) --- src/eynollah/eynollah.py | 277 ++++++++++++++++----------------------- 1 file changed, 112 insertions(+), 165 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 834ecf3..079cf8c 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -379,9 +379,14 @@ class Eynollah: self.model_reading_order = self.our_load_model(self.model_reading_order_dir) if self.ocr and self.tr: self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - #("microsoft/trocr-base-printed")#("microsoft/trocr-base-handwritten") - self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") + if torch.cuda.is_available(): + self.logger.info("Using GPU acceleration") + self.device = torch.device("cuda:0") + else: + self.logger.info("Using CPU processing") + self.device = torch.device("cpu") + #self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") + self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") elif self.ocr and not self.tr: model_ocr = load_model(self.model_ocr_dir , compile=False) @@ -4805,12 +4810,13 @@ class Eynollah: slopes_marginals, mid_point_of_page_width) #print(len(polygons_of_marginals), len(ordered_left_marginals), len(ordered_right_marginals), 'marginals ordred') + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( + contours_only_text_parent_d_ordered, index_by_text_par_con) + else: + contours_only_text_parent_d_ordered = None + if self.full_layout: - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d_ordered, index_by_text_par_con) - else: - contours_only_text_parent_d_ordered = None if self.light_version: fun = check_any_text_region_in_model_one_is_main_or_header_light else: @@ -4869,44 +4875,43 @@ class Eynollah: splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts, self.tables, self.right2left, logger=self.logger) + else: + contours_only_text_parent_h = [] + contours_only_text_parent_h_d_ordered = [] if self.plotter: self.plotter.write_images_into_directory(polygons_of_images, image_page) t_order = time.time() - if self.full_layout: - self.logger.info("Step 4/5: Reading Order Detection") - - if self.reading_order_machine_based: - self.logger.info("Using machine-based detection") - if self.right2left: - self.logger.info("Right-to-left mode enabled") - if self.headers_off: - self.logger.info("Headers ignored in reading order") + #if self.full_layout: + self.logger.info("Step 4/5: Reading Order Detection") - if self.reading_order_machine_based: - tror = time.time() - order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( - contours_only_text_parent, contours_only_text_parent_h, text_regions_p) + if self.reading_order_machine_based: + self.logger.info("Using machine-based detection") + if self.right2left: + self.logger.info("Right-to-left mode enabled") + if self.headers_off: + self.logger.info("Headers ignored in reading order") + + if self.reading_order_machine_based: + order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( + contours_only_text_parent, contours_only_text_parent_h, text_regions_p) + else: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + order_text_new, id_of_texts_tot = self.do_order_of_regions( + contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) else: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) - else: - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, - boxes_d, textline_mask_tot_d) - self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") + order_text_new, id_of_texts_tot = self.do_order_of_regions( + contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, + boxes_d, textline_mask_tot_d) + self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") - if self.ocr and not self.tr: - self.logger.info("Step 4.5/5: OCR Processing") - - if torch.cuda.is_available(): - self.logger.info("Using GPU acceleration") - else: - self.logger.info("Using CPU processing") - + if self.ocr: + self.logger.info("Step 4.5/5: OCR Processing") + + if not self.tr: gc.collect() + if len(all_found_textline_polygons)>0: ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons, self.prediction_model, @@ -4941,15 +4946,68 @@ class Eynollah: self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_drop = None + else: - ocr_all_textlines = None - ocr_all_textlines_marginals_left = None - ocr_all_textlines_marginals_right = None - ocr_all_textlines_h = None - ocr_all_textlines_drop = None + if self.light_version: + self.logger.info("Using light version OCR") + if self.textline_light: + self.logger.info("Using light text line detection for OCR") + self.logger.info("Processing text lines...") + + self.device.reset() + gc.collect() + + torch.cuda.empty_cache() + self.model_ocr.to(self.device) + + ind_tot = 0 + #cv2.imwrite('./img_out.png', image_page) + ocr_all_textlines = [] + for indexing, ind_poly_first in enumerate(all_found_textline_polygons): + ocr_textline_in_textregion = [] + for indexing2, ind_poly in enumerate(ind_poly_first): + if not (self.textline_light or self.curved_line): + ind_poly = copy.deepcopy(ind_poly) + box_ind = all_box_coord[indexing] + #print(ind_poly,np.shape(ind_poly), 'ind_poly') + #print(box_ind) + ind_poly = return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) + #print(ind_poly_copy) + ind_poly[ind_poly<0] = 0 + x, y, w, h = cv2.boundingRect(ind_poly) + #print(ind_poly_copy, np.shape(ind_poly_copy)) + #print(x, y, w, h, h/float(w),'ratio') + h2w_ratio = h/float(w) + mask_poly = np.zeros(image_page.shape) + if not self.light_version: + img_poly_on_img = np.copy(image_page) + else: + img_poly_on_img = np.copy(img_bin_light) + mask_poly = cv2.fillPoly(mask_poly, pts=[ind_poly], color=(1, 1, 1)) + + if self.textline_light: + mask_poly = cv2.dilate(mask_poly, KERNEL, iterations=1) + img_poly_on_img[:,:,0][mask_poly[:,:,0] ==0] = 255 + img_poly_on_img[:,:,1][mask_poly[:,:,0] ==0] = 255 + img_poly_on_img[:,:,2][mask_poly[:,:,0] ==0] = 255 + + img_croped = img_poly_on_img[y:y+h, x:x+w, :] + #cv2.imwrite('./extracted_lines/'+str(ind_tot)+'.jpg', img_croped) + text_ocr = self.return_ocr_of_textline_without_common_section( + img_croped, self.model_ocr, self.processor, self.device, w, h2w_ratio, ind_tot) + ocr_textline_in_textregion.append(text_ocr) + ind_tot = ind_tot +1 + ocr_all_textlines.append(ocr_textline_in_textregion) + else: + ocr_all_textlines = None + ocr_all_textlines_marginals_left = None + ocr_all_textlines_marginals_right = None + ocr_all_textlines_h = None + ocr_all_textlines_drop = None - self.logger.info("Step 5/5: Output Generation") - + self.logger.info("Step 5/5: Output Generation") + + if self.full_layout: pcgts = self.writer.build_pagexml_full_layout( contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, @@ -4962,129 +5020,18 @@ class Eynollah: ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, ocr_all_textlines_drop, conf_contours_textregions, conf_contours_textregions_h) - - return pcgts - - contours_only_text_parent_h = [] - self.logger.info("Step 4/5: Reading Order Detection") - - if self.reading_order_machine_based: - self.logger.info("Using machine-based detection") - if self.right2left: - self.logger.info("Right-to-left mode enabled") - if self.headers_off: - self.logger.info("Headers ignored in reading order") - - if self.reading_order_machine_based: - order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( - contours_only_text_parent, contours_only_text_parent_h, text_regions_p) else: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) - else: - contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d_ordered, index_by_text_par_con) - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) - - if self.ocr and self.tr: - self.logger.info("Step 4.5/5: OCR Processing") - if torch.cuda.is_available(): - self.logger.info("Using GPU acceleration") - else: - self.logger.info("Using CPU processing") - if self.light_version: - self.logger.info("Using light version OCR") - if self.textline_light: - self.logger.info("Using light text line detection for OCR") - self.logger.info("Processing text lines...") + pcgts = self.writer.build_pagexml_no_full_layout( + txt_con_org, page_coord, order_text_new, id_of_texts_tot, + all_found_textline_polygons, all_box_coord, polygons_of_images, + polygons_of_marginals_left, polygons_of_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, contours_tables, ocr_all_textlines, + ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, + conf_contours_textregions) - device = cuda.get_current_device() - device.reset() - gc.collect() - model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) - device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") - torch.cuda.empty_cache() - model_ocr.to(device) - - ind_tot = 0 - #cv2.imwrite('./img_out.png', image_page) - ocr_all_textlines = [] - for indexing, ind_poly_first in enumerate(all_found_textline_polygons): - ocr_textline_in_textregion = [] - for indexing2, ind_poly in enumerate(ind_poly_first): - if not (self.textline_light or self.curved_line): - ind_poly = copy.deepcopy(ind_poly) - box_ind = all_box_coord[indexing] - #print(ind_poly,np.shape(ind_poly), 'ind_poly') - #print(box_ind) - ind_poly = return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) - #print(ind_poly_copy) - ind_poly[ind_poly<0] = 0 - x, y, w, h = cv2.boundingRect(ind_poly) - #print(ind_poly_copy, np.shape(ind_poly_copy)) - #print(x, y, w, h, h/float(w),'ratio') - h2w_ratio = h/float(w) - mask_poly = np.zeros(image_page.shape) - if not self.light_version: - img_poly_on_img = np.copy(image_page) - else: - img_poly_on_img = np.copy(img_bin_light) - mask_poly = cv2.fillPoly(mask_poly, pts=[ind_poly], color=(1, 1, 1)) - - if self.textline_light: - mask_poly = cv2.dilate(mask_poly, KERNEL, iterations=1) - img_poly_on_img[:,:,0][mask_poly[:,:,0] ==0] = 255 - img_poly_on_img[:,:,1][mask_poly[:,:,0] ==0] = 255 - img_poly_on_img[:,:,2][mask_poly[:,:,0] ==0] = 255 - - img_croped = img_poly_on_img[y:y+h, x:x+w, :] - #cv2.imwrite('./extracted_lines/'+str(ind_tot)+'.jpg', img_croped) - text_ocr = self.return_ocr_of_textline_without_common_section( - img_croped, model_ocr, processor, device, w, h2w_ratio, ind_tot) - ocr_textline_in_textregion.append(text_ocr) - ind_tot = ind_tot +1 - ocr_all_textlines.append(ocr_textline_in_textregion) - - elif self.ocr and not self.tr: - gc.collect() - if len(all_found_textline_polygons)>0: - ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - - if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: - ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons_marginals_left, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - - if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: - ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons_marginals_right, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - - else: - ocr_all_textlines = None - ocr_all_textlines_marginals_left = None - ocr_all_textlines_marginals_right = None - self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") - - self.logger.info("Step 5/5: Output Generation") - self.logger.info("Generating PAGE-XML output") - - pcgts = self.writer.build_pagexml_no_full_layout( - txt_con_org, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, all_box_coord, polygons_of_images, - polygons_of_marginals_left, polygons_of_marginals_right, - all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, - all_box_coord_marginals_left, all_box_coord_marginals_right, - slopes, slopes_marginals_left, slopes_marginals_right, - cont_page, polygons_seplines, contours_tables, ocr_all_textlines, - ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, - conf_contours_textregions) - return pcgts From a1904fa660e7cb79ba9b4d8fc7df5befc41072f1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 17:44:12 +0200 Subject: [PATCH 332/374] tests: cover layout with OCR in various modes --- tests/test_run.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index 59e5099..d69f021 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -24,14 +24,18 @@ MODELS_BIN = environ.get('MODELS_BIN', str(testdir.joinpath('..', 'default-2021- "options", [ [], # defaults - ["--allow_scaling", "--curved-line"], + #["--allow_scaling", "--curved-line"], ["--allow_scaling", "--curved-line", "--full-layout"], ["--allow_scaling", "--curved-line", "--full-layout", "--reading_order_machine_based"], ["--allow_scaling", "--curved-line", "--full-layout", "--reading_order_machine_based", "--textline_light", "--light_version"], # -ep ... # -eoi ... - # --do_ocr + ["--do_ocr"], + ["--do_ocr", "--light_version", "--textline_light"], + ["--do_ocr", "--transformer_ocr"], + #["--do_ocr", "--transformer_ocr", "--light_version", "--textline_light"], + ["--do_ocr", "--transformer_ocr", "--light_version", "--textline_light", "--full-layout"], # --skip_layout_and_reading_order ], ids=str) def test_run_eynollah_layout_filename(tmp_path, pytestconfig, caplog, options): From 23535998f7532942d481f3729682969e19c228b6 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Mon, 6 Oct 2025 21:27:21 +0200 Subject: [PATCH 333/374] tests: symlink OCR models into layout model directory (so layout with OCR options works with our split model packages) --- Makefile | 19 +++++++++++-------- tests/test_run.py | 3 ++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 357aa47..5d190b2 100644 --- a/Makefile +++ b/Makefile @@ -90,26 +90,29 @@ deps-test: $(OCR_MODELNAME) endif deps-test: $(BIN_MODELNAME) $(SEG_MODELNAME) $(PIP) install -r requirements-test.txt +ifeq (OCR,$(findstring OCR, $(EXTRAS))) + ln -s $(OCR_MODELNAME)/* $(SEG_MODELNAME)/ +endif smoke-test: TMPDIR != mktemp -d smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif # layout analysis: - eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/models_layout_v0_5_0 + eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/$(SEG_MODELNAME) fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$(basename $( Date: Tue, 7 Oct 2025 00:54:25 +0200 Subject: [PATCH 334/374] CI: run deps-test with OCR extra so symlink rule fires --- .github/workflows/test-eynollah.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index 9d5b2c8..7c3f5ae 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -65,7 +65,7 @@ jobs: run: | python -m pip install --upgrade pip make install-dev EXTRAS=OCR,plotting - make deps-test + make deps-test EXTRAS=OCR,plotting - name: Test with pytest run: make coverage PYTEST_ARGS="-vv --junitxml=pytest.xml" - name: Get coverage results diff --git a/Makefile b/Makefile index 5d190b2..618b1f9 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ endif deps-test: $(BIN_MODELNAME) $(SEG_MODELNAME) $(PIP) install -r requirements-test.txt ifeq (OCR,$(findstring OCR, $(EXTRAS))) - ln -s $(OCR_MODELNAME)/* $(SEG_MODELNAME)/ + ln -rs $(OCR_MODELNAME)/* $(SEG_MODELNAME)/ endif smoke-test: TMPDIR != mktemp -d From d53f829dfd0b26e4738915b24ffe4256796c6eb4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:06:57 +0200 Subject: [PATCH 335/374] filter_contours_inside_a_bigger_one: fix edge case in 81827c29 --- src/eynollah/eynollah.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 079cf8c..271779f 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4068,7 +4068,9 @@ class Eynollah: for textregion_index_to_del in textline_in_textregion_index_to_del: contours[textregion_index_to_del] = list(np.delete( contours[textregion_index_to_del], - textline_in_textregion_index_to_del[textregion_index_to_del])) + textline_in_textregion_index_to_del[textregion_index_to_del], + # needed so numpy does not flatten the entire result when 0 left + axis=0)) return contours From 2e907875c12b4f22c650c109558917479e0ec3ae Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:32:06 +0200 Subject: [PATCH 336/374] get_text_region_boxes_by_given_contours: simplify --- src/eynollah/eynollah.py | 4 ++-- src/eynollah/utils/contour.py | 10 ++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 271779f..06be910 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4726,8 +4726,8 @@ class Eynollah: txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) #print("text region early 4 in %.1fs", time.time() - t0) - boxes_text, _ = get_text_region_boxes_by_given_contours(contours_only_text_parent) - boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) + boxes_text = get_text_region_boxes_by_given_contours(contours_only_text_parent) + boxes_marginals = get_text_region_boxes_by_given_contours(polygons_of_marginals) #print("text region early 5 in %.1fs", time.time() - t0) ## birdan sora chock chakir if not self.curved_line: diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 22a6f50..fb4bbd0 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -36,14 +36,8 @@ def find_contours_mean_y_diff(contours_main): return np.mean(np.diff(np.sort(np.array(cy_main)))) def get_text_region_boxes_by_given_contours(contours): - boxes = [] - contours_new = [] - for jj in range(len(contours)): - box = cv2.boundingRect(contours[jj]) - boxes.append(box) - contours_new.append(contours[jj]) - - return boxes, contours_new + return [cv2.boundingRect(contour) + for contour in contours] def filter_contours_area_of_image(image, contours, hierarchy, max_area=1.0, min_area=0.0, dilate=0): found_polygons_early = [] From dfdc70537530b55f77b5232ae3cfa1fc8357eed0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:33:06 +0200 Subject: [PATCH 337/374] do_work_of_slopes: rm unused old variant --- src/eynollah/eynollah.py | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 06be910..2431a3b 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -108,7 +108,6 @@ from .utils.utils_ocr import ( get_contours_and_bounding_boxes ) from .utils.separate_lines import ( - textline_contours_postprocessing, separate_lines_new2, return_deskew_slop, do_work_of_slopes_new, @@ -2062,43 +2061,6 @@ class Eynollah: (prediction_textline_longshot_true_size[:, :, 0]==1).astype(np.uint8)) - def do_work_of_slopes(self, q, poly, box_sub, boxes_per_process, textline_mask_tot, contours_per_process): - self.logger.debug('enter do_work_of_slopes') - slope_biggest = 0 - slopes_sub = [] - boxes_sub_new = [] - poly_sub = [] - for mv in range(len(boxes_per_process)): - crop_img, _ = crop_image_inside_box(boxes_per_process[mv], textline_mask_tot) - crop_img = cv2.erode(crop_img, KERNEL, iterations=2) - try: - textline_con, hierarchy = return_contours_of_image(crop_img) - textline_con_fil = filter_contours_area_of_image(crop_img, textline_con, hierarchy, - max_area=1, min_area=0.0008) - y_diff_mean = find_contours_mean_y_diff(textline_con_fil) - sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) - crop_img[crop_img > 0] = 1 - slope_corresponding_textregion = return_deskew_slop(crop_img, sigma_des, - logger=self.logger, plotter=self.plotter) - except Exception as why: - self.logger.error(why) - slope_corresponding_textregion = MAX_SLOPE - - if slope_corresponding_textregion == MAX_SLOPE: - slope_corresponding_textregion = slope_biggest - slopes_sub.append(slope_corresponding_textregion) - - cnt_clean_rot = textline_contours_postprocessing( - crop_img, slope_corresponding_textregion, contours_per_process[mv], boxes_per_process[mv]) - - poly_sub.append(cnt_clean_rot) - boxes_sub_new.append(boxes_per_process[mv]) - - q.put(slopes_sub) - poly.put(poly_sub) - box_sub.put(boxes_sub_new) - self.logger.debug('exit do_work_of_slopes') - def get_regions_light_v_extract_only_images(self,img,is_image_enhanced, num_col_classifier): self.logger.debug("enter get_regions_extract_images_only") erosion_hurts = False From 0a80cd5dffc7e5c28f41330da8d2f1255ac66e88 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:37:05 +0200 Subject: [PATCH 338/374] avoid unnecessary 3-channel conversions: for tables, too --- src/eynollah/eynollah.py | 155 ++++++++++++++++----------------------- 1 file changed, 65 insertions(+), 90 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 2431a3b..70a8a17 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -930,10 +930,8 @@ class Eynollah: img_w = img.shape[1] prediction_true = np.zeros((img_h, img_w, 3)) mask_true = np.zeros((img_h, img_w)) - nxf = img_w / float(width_mid) - nyf = img_h / float(height_mid) - nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) - nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + nxf = math.ceil(img_w / float(width_mid)) + nyf = math.ceil(img_h / float(height_mid)) list_i_s = [] list_j_s = [] @@ -946,18 +944,10 @@ class Eynollah: img_patch = np.zeros((n_batch_inference, img_height_model, img_width_model, 3)) for i in range(nxf): for j in range(nyf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - else: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - if j == 0: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - else: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model if index_x_u > img_w: index_x_u = img_w index_x_d = img_w - img_width_model @@ -2600,23 +2590,20 @@ class Eynollah: self, layout, table_prediction_early, pixel_table, num_col_classifier): layout_org = np.copy(layout) - layout_org[:,:,0][layout_org[:,:,0]==pixel_table] = 0 - layout = (layout[:,:,0]==pixel_table)*1 - - layout = layout.astype(np.uint8) + layout_org[layout_org == pixel_table] = 0 + layout = (layout == pixel_table).astype(np.uint8) * 1 _, thresh = cv2.threshold(layout, 0, 255, 0) contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - cnt_size = np.array([cv2.contourArea(contours[j]) - for j in range(len(contours))]) + cnt_size = np.array([cv2.contourArea(cnt) for cnt in contours]) contours_new = [] - for i in range(len(contours)): - x, y, w, h = cv2.boundingRect(contours[i]) + for i, contour in enumerate(contours): + x, y, w, h = cv2.boundingRect(contour) iou = cnt_size[i] /float(w*h) *100 if iou<80: layout_contour = np.zeros(layout_org.shape[:2]) - layout_contour = cv2.fillPoly(layout_contour, pts=[contours[i]] ,color=1) + layout_contour = cv2.fillPoly(layout_contour, pts=[contour] ,color=1) layout_contour_sum = layout_contour.sum(axis=0) layout_contour_sum_diff = np.diff(layout_contour_sum) @@ -2648,26 +2635,26 @@ class Eynollah: #print(iou_in,'iou_in_in1') if iou_in>30: - layout_org= cv2.fillPoly(layout_org, pts=[contours_sep[ji]], color=3 * (pixel_table,)) + layout_org = cv2.fillPoly(layout_org, pts=[contours_sep[ji]], color=pixel_table) else: pass else: - layout_org= cv2.fillPoly(layout_org, pts=[contours_sep[ji]], color=3 * (pixel_table,)) + layout_org= cv2.fillPoly(layout_org, pts=[contours_sep[ji]], color=pixel_table) else: - contours_new.append(contours[i]) + contours_new.append(contour) if num_col_classifier>=2: - only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1])) - only_recent_contour_image= cv2.fillPoly(only_recent_contour_image,pts=[contours[i]] ,color=(1,1,1)) + only_recent_contour_image = np.zeros(layout.shape[:2]) + only_recent_contour_image = cv2.fillPoly(only_recent_contour_image, pts=[contour],color=1) table_pixels_masked_from_early_pre = only_recent_contour_image * table_prediction_early iou_in = 100. * table_pixels_masked_from_early_pre.sum() / only_recent_contour_image.sum() #print(iou_in,'iou_in') if iou_in>30: - layout_org= cv2.fillPoly(layout_org, pts=[contours[i]], color=3 * (pixel_table,)) + layout_org = cv2.fillPoly(layout_org, pts=[contour], color=pixel_table) else: pass else: - layout_org= cv2.fillPoly(layout_org, pts=[contours[i]], color=3 * (pixel_table,)) + layout_org = cv2.fillPoly(layout_org, pts=[contour], color=pixel_table) return layout_org, contours_new @@ -2714,16 +2701,10 @@ class Eynollah: pass boxes = np.array(boxes, dtype=int) # to be on the safe side - img_comm_e = np.zeros(image_revised_1.shape) - img_comm = np.repeat(img_comm_e[:, :, np.newaxis], 3, axis=2) - + img_comm = np.zeros(image_revised_1.shape, dtype=np.uint8) for indiv in np.unique(image_revised_1): - image_col=(image_revised_1==indiv)*255 - img_comm_in=np.repeat(image_col[:, :, np.newaxis], 3, axis=2) - img_comm_in=img_comm_in.astype(np.uint8) - - imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + image_col = (image_revised_1 == indiv).astype(np.uint8) * 255 + _, thresh = cv2.threshold(image_col, 0, 255, 0) contours,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) if indiv==pixel_table: @@ -2733,35 +2714,27 @@ class Eynollah: main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area=1, min_area=min_area) - img_comm = cv2.fillPoly(img_comm, pts = main_contours, color = (indiv, indiv, indiv)) - img_comm = img_comm.astype(np.uint8) + img_comm = cv2.fillPoly(img_comm, pts=main_contours, color=indiv) if not self.isNaN(slope_mean_hor): - image_revised_last = np.zeros((image_regions_eraly_p.shape[0], image_regions_eraly_p.shape[1],3)) + image_revised_last = np.zeros(image_regions_eraly_p.shape[:2]) for i in range(len(boxes)): box_ys = slice(*boxes[i][2:4]) box_xs = slice(*boxes[i][0:2]) image_box = img_comm[box_ys, box_xs] try: - image_box_tabels_1=(image_box[:,:,0]==pixel_table)*1 + image_box_tabels_1 = (image_box == pixel_table) * 1 contours_tab,_=return_contours_of_image(image_box_tabels_1) contours_tab=filter_contours_area_of_image_tables(image_box_tabels_1,contours_tab,_,1,0.003) - image_box_tabels_1=(image_box[:,:,0]==pixel_line)*1 + image_box_tabels_1 = (image_box == pixel_line).astype(np.uint8) * 1 + image_box_tabels_and_m_text = ( (image_box == pixel_table) | + (image_box == 1) ).astype(np.uint8) * 1 - image_box_tabels_and_m_text=( (image_box[:,:,0]==pixel_table) | (image_box[:,:,0]==1) )*1 - image_box_tabels_and_m_text=image_box_tabels_and_m_text.astype(np.uint8) + image_box_tabels_1 = cv2.dilate(image_box_tabels_1, KERNEL, iterations=5) - image_box_tabels_1=image_box_tabels_1.astype(np.uint8) - image_box_tabels_1 = cv2.dilate(image_box_tabels_1,KERNEL,iterations = 5) - - contours_table_m_text,_=return_contours_of_image(image_box_tabels_and_m_text) - image_box_tabels=np.repeat(image_box_tabels_1[:, :, np.newaxis], 3, axis=2) - - image_box_tabels=image_box_tabels.astype(np.uint8) - imgray = cv2.cvtColor(image_box_tabels, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_line,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + contours_table_m_text, _ = return_contours_of_image(image_box_tabels_and_m_text) + _, thresh = cv2.threshold(image_box_tabels_1, 0, 255, 0) + contours_line, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) y_min_main_line ,y_max_main_line=find_features_of_contours(contours_line) y_min_main_tab ,y_max_main_tab=find_features_of_contours(contours_tab) @@ -2793,18 +2766,20 @@ class Eynollah: y_max_main_tab[i_t] < y_min_main_line[i_l] and y_min_main_tab[i_t] < y_min_main_line[i_l]): pass - elif np.abs(y_max_main_line[i_l]-y_min_main_line[i_l])<100: + elif abs(y_max_main_line[i_l] - y_min_main_line[i_l]) < 100: pass else: - y_up_tab.append(np.min([y_min_main_line[i_l], y_min_main_tab[i_t] ]) ) - y_down_tab.append( np.max([ y_max_main_line[i_l],y_max_main_tab[i_t] ]) ) + y_up_tab.append(min([y_min_main_line[i_l], + y_min_main_tab[i_t]])) + y_down_tab.append(max([y_max_main_line[i_l], + y_max_main_tab[i_t]])) if len(y_up_tab)==0: y_up_tabs.append(y_min_main_tab[i_t]) y_down_tabs.append(y_max_main_tab[i_t]) else: - y_up_tabs.append(np.min(y_up_tab)) - y_down_tabs.append(np.max(y_down_tab)) + y_up_tabs.append(min(y_up_tab)) + y_down_tabs.append(max(y_down_tab)) else: y_down_tabs=[] y_up_tabs=[] @@ -2814,7 +2789,7 @@ class Eynollah: y_up_tabs=[] for ii in range(len(y_up_tabs)): - image_box[y_up_tabs[ii]:y_down_tabs[ii],:,0]=pixel_table + image_box[y_up_tabs[ii]:y_down_tabs[ii]] = pixel_table image_revised_last[box_ys, box_xs] = image_box else: @@ -2825,14 +2800,14 @@ class Eynollah: image_revised_last[box_ys, box_xs] = image_box if num_col_classifier==1: - img_tables_col_1 = (image_revised_last[:,:,0] == pixel_table).astype(np.uint8) + img_tables_col_1 = (image_revised_last == pixel_table).astype(np.uint8) contours_table_col1, _ = return_contours_of_image(img_tables_col_1) _,_ ,_ , _, y_min_tab_col1 ,y_max_tab_col1, _= find_new_features_of_contours(contours_table_col1) if len(y_min_tab_col1)>0: for ijv in range(len(y_min_tab_col1)): - image_revised_last[int(y_min_tab_col1[ijv]):int(y_max_tab_col1[ijv]),:,:]=pixel_table + image_revised_last[int(y_min_tab_col1[ijv]):int(y_max_tab_col1[ijv])] = pixel_table return image_revised_last def get_tables_from_model(self, img, num_col_classifier): @@ -3200,7 +3175,7 @@ class Eynollah: pass else: text_regions_p_tables = np.copy(text_regions_p) - text_regions_p_tables[:,:][(table_prediction[:,:] == 1)] = 10 + text_regions_p_tables[(table_prediction == 1)] = 10 pixel_line = 3 img_revised_tab2 = self.add_tables_heuristic_to_layout( text_regions_p_tables, boxes, 0, splitter_y_new, peaks_neg_tot_tables, text_regions_p_tables, @@ -3221,8 +3196,8 @@ class Eynollah: pass else: text_regions_p_tables = np.copy(text_regions_p_1_n) - text_regions_p_tables =np.round(text_regions_p_tables) - text_regions_p_tables[:,:][(text_regions_p_tables[:,:] != 3) & (table_prediction_n[:,:] == 1)] = 10 + text_regions_p_tables = np.round(text_regions_p_tables) + text_regions_p_tables[(text_regions_p_tables != 3) & (table_prediction_n == 1)] = 10 pixel_line = 3 img_revised_tab2 = self.add_tables_heuristic_to_layout( @@ -3242,21 +3217,21 @@ class Eynollah: if self.tables: if self.light_version: - text_regions_p[:,:][table_prediction[:,:]==1] = 10 - img_revised_tab=text_regions_p[:,:] + text_regions_p[table_prediction == 1] = 10 + img_revised_tab = text_regions_p[:,:] else: if np.abs(slope_deskew) < SLOPE_THRESHOLD: - img_revised_tab = np.copy(img_revised_tab2[:,:,0]) - img_revised_tab[:,:][(text_regions_p[:,:] == 1) & (img_revised_tab[:,:] != 10)] = 1 + img_revised_tab = np.copy(img_revised_tab2) + img_revised_tab[(text_regions_p == 1) & (img_revised_tab != 10)] = 1 else: - img_revised_tab = np.copy(text_regions_p[:,:]) - img_revised_tab[:,:][img_revised_tab[:,:] == 10] = 0 - img_revised_tab[:,:][img_revised_tab2_d_rotated[:,:,0] == 10] = 10 + img_revised_tab = np.copy(text_regions_p) + img_revised_tab[img_revised_tab == 10] = 0 + img_revised_tab[img_revised_tab2_d_rotated == 10] = 10 - text_regions_p[:,:][text_regions_p[:,:]==10] = 0 - text_regions_p[:,:][img_revised_tab[:,:]==10] = 10 + text_regions_p[text_regions_p == 10] = 0 + text_regions_p[img_revised_tab == 10] = 10 else: - img_revised_tab=text_regions_p[:,:] + img_revised_tab = text_regions_p[:,:] #img_revised_tab = text_regions_p[:, :] if self.light_version: polygons_of_images = return_contours_of_interested_region(text_regions_p, 2) @@ -3386,7 +3361,7 @@ class Eynollah: num_col_classifier, erosion_hurts, self.tables, self.right2left) text_regions_p_tables = np.copy(text_regions_p_1_n) text_regions_p_tables = np.round(text_regions_p_tables) - text_regions_p_tables[:,:][(text_regions_p_tables[:,:]!=3) & (table_prediction_n[:,:]==1)] = 10 + text_regions_p_tables[(text_regions_p_tables != 3) & (table_prediction_n == 1)] = 10 pixel_line = 3 img_revised_tab2 = self.add_tables_heuristic_to_layout( @@ -3405,17 +3380,17 @@ class Eynollah: text_regions_p.shape[1]) if np.abs(slope_deskew) < 0.13: - img_revised_tab = np.copy(img_revised_tab2[:,:,0]) + img_revised_tab = np.copy(img_revised_tab2) else: - img_revised_tab = np.copy(text_regions_p[:,:]) - img_revised_tab[:,:][img_revised_tab[:,:] == 10] = 0 - img_revised_tab[:,:][img_revised_tab2_d_rotated[:,:,0] == 10] = 10 + img_revised_tab = np.copy(text_regions_p) + img_revised_tab[img_revised_tab == 10] = 0 + img_revised_tab[img_revised_tab2_d_rotated == 10] = 10 - ##img_revised_tab=img_revised_tab2[:,:,0] - #img_revised_tab=text_regions_p[:,:] - text_regions_p[:,:][text_regions_p[:,:]==10] = 0 - text_regions_p[:,:][img_revised_tab[:,:]==10] = 10 - #img_revised_tab[img_revised_tab2[:,:,0]==10] =10 + ##img_revised_tab = img_revised_tab2[:,:] + #img_revised_tab = text_regions_p[:,:] + text_regions_p[text_regions_p == 10] = 0 + text_regions_p[img_revised_tab == 10] = 10 + #img_revised_tab[img_revised_tab2 == 10] = 10 pixel_img = 4 min_area_mar = 0.00001 From fd43e78442251c552faafeffe02256023ae1a806 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:42:36 +0200 Subject: [PATCH 339/374] filter_contours_without_textline_inside: simplify - np.delete in index array instead of contour lists - yield actual resulting indices --- src/eynollah/eynollah.py | 77 ++++------------------------------------ 1 file changed, 6 insertions(+), 71 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 70a8a17..6cc8b1b 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4040,79 +4040,23 @@ class Eynollah: self, contours, text_con_org, contours_textline, contours_only_text_parent_d_ordered, conf_contours_textregions): - ###contours_txtline_of_all_textregions = [] - ###for jj in range(len(contours_textline)): - ###contours_txtline_of_all_textregions = contours_txtline_of_all_textregions + contours_textline[jj] - ###M_main_textline = [cv2.moments(contours_txtline_of_all_textregions[j]) - ### for j in range(len(contours_txtline_of_all_textregions))] - ###cx_main_textline = [(M_main_textline[j]["m10"] / (M_main_textline[j]["m00"] + 1e-32)) - ### for j in range(len(M_main_textline))] - ###cy_main_textline = [(M_main_textline[j]["m01"] / (M_main_textline[j]["m00"] + 1e-32)) - ### for j in range(len(M_main_textline))] - - ###M_main = [cv2.moments(contours[j]) for j in range(len(contours))] - ###cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - ###cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - - ###contours_with_textline = [] - ###for ind_tr, con_tr in enumerate(contours): - ###results = [cv2.pointPolygonTest(con_tr, - ### (cx_main_textline[index_textline_con], - ### cy_main_textline[index_textline_con]), - ### False) - ### for index_textline_con in range(len(contours_txtline_of_all_textregions)) ] - ###results = np.array(results) - ###if np.any(results==1): - ###contours_with_textline.append(con_tr) - - textregion_index_to_del = set() - for index_textregion, textlines_textregion in enumerate(contours_textline): - if len(textlines_textregion) == 0: - textregion_index_to_del.add(index_textregion) + assert len(contours_par) == len(contours_textline) + indices = np.arange(len(contours_textline)) + indices = np.delete(indices, np.flatnonzero([len(lines) == 0 for lines in contours_textline])) def filterfun(lis): if len(lis) == 0: return [] - if len(textregion_index_to_del) == 0: - return lis - return list(np.delete(lis, list(textregion_index_to_del))) + return list(np.array(lis)[indices]) return (filterfun(contours), filterfun(text_con_org), filterfun(conf_contours_textregions), filterfun(contours_textline), filterfun(contours_only_text_parent_d_ordered), - np.arange(len(contours) - len(textregion_index_to_del))) + indices + ) - def delete_regions_without_textlines( - self, slopes, all_found_textline_polygons, boxes_text, txt_con_org, - contours_only_text_parent, index_by_text_par_con): - - slopes_rem = [] - all_found_textline_polygons_rem = [] - boxes_text_rem = [] - txt_con_org_rem = [] - contours_only_text_parent_rem = [] - index_by_text_par_con_rem = [] - - for i, ind_con in enumerate(all_found_textline_polygons): - if len(ind_con): - all_found_textline_polygons_rem.append(ind_con) - slopes_rem.append(slopes[i]) - boxes_text_rem.append(boxes_text[i]) - txt_con_org_rem.append(txt_con_org[i]) - contours_only_text_parent_rem.append(contours_only_text_parent[i]) - index_by_text_par_con_rem.append(index_by_text_par_con[i]) - - index_sort = np.argsort(index_by_text_par_con_rem) - indexes_new = np.array(range(len(index_by_text_par_con_rem))) - - index_by_text_par_con_rem_sort = [indexes_new[index_sort==j][0] - for j in range(len(index_by_text_par_con_rem))] - - return (slopes_rem, all_found_textline_polygons_rem, boxes_text_rem, txt_con_org_rem, - contours_only_text_parent_rem, index_by_text_par_con_rem_sort) - def separate_marginals_to_left_and_right_and_order_from_top_to_down( self, polygons_of_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes_marginals, mid_point_of_page_width): @@ -4679,15 +4623,6 @@ class Eynollah: polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_org, boxes_marginals, slope_deskew) - #slopes, all_found_textline_polygons, boxes_text, txt_con_org, \ - # contours_only_text_parent, index_by_text_par_con = \ - # self.delete_regions_without_textlines(slopes, all_found_textline_polygons, - # boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con) - #slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, \ - # polygons_of_marginals, polygons_of_marginals, _ = \ - # self.delete_regions_without_textlines(slopes_marginals, all_found_textline_polygons_marginals, - # boxes_marginals, polygons_of_marginals, polygons_of_marginals, - # np.array(range(len(polygons_of_marginals)))) all_found_textline_polygons = dilate_textline_contours( all_found_textline_polygons) all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( From 02a347a48a972de49c4b098f454a9a16cc4ee4fc Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:47:34 +0200 Subject: [PATCH 340/374] no more need to rm from `contours_only_text_parent_d_ordered` now --- src/eynollah/eynollah.py | 16 ++-------------- src/eynollah/utils/__init__.py | 8 ++++---- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 6cc8b1b..c4a6600 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4437,6 +4437,8 @@ class Eynollah: ###min_con_area = 0.000005 contours_only_text, hir_on_text = return_contours_of_image(text_only) contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) + contours_only_text_parent_d_ordered = [] + contours_only_text_parent_d = [] if len(contours_only_text_parent) > 0: areas_tot_text = np.prod(text_only.shape) areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) @@ -4558,15 +4560,6 @@ class Eynollah: # plt.subplot(2, 2, 2, title="result contours") # plt.imshow(img4) # plt.show() - else: - contours_only_text_parent_d_ordered = [] - contours_only_text_parent_d = [] - contours_only_text_parent = [] - - else: - contours_only_text_parent_d_ordered = [] - contours_only_text_parent_d = [] - #contours_only_text_parent = [] if not len(contours_only_text_parent): # stop early @@ -4684,11 +4677,6 @@ class Eynollah: slopes_marginals, mid_point_of_page_width) #print(len(polygons_of_marginals), len(ordered_left_marginals), len(ordered_right_marginals), 'marginals ordred') - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d_ordered, index_by_text_par_con) - else: - contours_only_text_parent_d_ordered = None if self.full_layout: if self.light_version: diff --git a/src/eynollah/utils/__init__.py b/src/eynollah/utils/__init__.py index ebf78fe..5ccb2af 100644 --- a/src/eynollah/utils/__init__.py +++ b/src/eynollah/utils/__init__.py @@ -938,7 +938,7 @@ def check_any_text_region_in_model_one_is_main_or_header( if (pixels_header>=pixels_main) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ]=2 contours_only_text_parent_head.append(con) - if contours_only_text_parent_d_ordered is not None: + if len(contours_only_text_parent_d_ordered): contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_head.append(all_box_coord[ii]) slopes_head.append(slopes[ii]) @@ -948,7 +948,7 @@ def check_any_text_region_in_model_one_is_main_or_header( regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ]=1 contours_only_text_parent_main.append(con) conf_contours_main.append(conf_contours[ii]) - if contours_only_text_parent_d_ordered is not None: + if len(contours_only_text_parent_d_ordered): contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_main.append(all_box_coord[ii]) slopes_main.append(slopes[ii]) @@ -1033,7 +1033,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light( regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ] = 2 contours_only_text_parent_head.append(contours_only_text_parent[ii]) conf_contours_head.append(None) # why not conf_contours[ii], too? - if contours_only_text_parent_d_ordered is not None: + if len(contours_only_text_parent_d_ordered): contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_head.append(all_box_coord[ii]) slopes_head.append(slopes[ii]) @@ -1043,7 +1043,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light( regions_model_1[:,:][(regions_model_1[:,:]==1) & (img == 255) ] = 1 contours_only_text_parent_main.append(contours_only_text_parent[ii]) conf_contours_main.append(conf_contours[ii]) - if contours_only_text_parent_d_ordered is not None: + if len(contours_only_text_parent_d_ordered): contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_main.append(all_box_coord[ii]) slopes_main.append(slopes[ii]) From d88ca18eec8f1a4def371848c218b817fdb728a1 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 22:53:30 +0200 Subject: [PATCH 341/374] get/do_work_of_slopes etc.: reduce call/return signatures - `get_textregion_contours_in_org_image_light`: no more need to also return unchanged contours here (see 41cc38c5); therefore - `txt_con_org`: no more need for this (now mere alias to `contours_only_text_parent`); also - `index_by_text_par_con`: no more need for this (see prev. commit), so do not pass/return - `get_slopes_and_deskew_*`: do not pass `contours_only_text` (where not used) - `get_slopes_and_deskew_*`: do not return unchanged contours, boxes - `do_work_of_slopes_*`: adapt respectively --- src/eynollah/eynollah.py | 98 +++++++++++++--------------- src/eynollah/utils/contour.py | 4 +- src/eynollah/utils/separate_lines.py | 12 ++-- 3 files changed, 54 insertions(+), 60 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index c4a6600..ec68bcd 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -879,7 +879,7 @@ class Eynollah: thresholding_for_fl_light_version=False, threshold_art_class_textline=0.1): - self.logger.debug("enter do_prediction") + self.logger.debug("enter do_prediction (patches=%d)", patches) img_height_model = model.layers[-1].output_shape[1] img_width_model = model.layers[-1].output_shape[2] @@ -1856,7 +1856,7 @@ class Eynollah: return sorted_textlines - def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): + def get_slopes_and_deskew_new_light2(self, contours_par, textline_mask_tot, boxes, slope_deskew): polygons_of_textlines = return_contours_of_interested_region(textline_mask_tot,1,0.00001) cx_main_tot, cy_main_tot = find_center_of_contours(polygons_of_textlines) @@ -1889,16 +1889,12 @@ class Eynollah: all_box_coord.append(crop_coor) return (all_found_textline_polygons, - boxes, - contours, - contours_par, all_box_coord, - np.array(range(len(contours_par))), slopes) def get_slopes_and_deskew_new_light(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): if not len(contours): - return [], [], [], [], [], [], [] + return [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_light") with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: results = self.executor.map(partial(do_work_of_slopes_new_light, @@ -1906,15 +1902,15 @@ class Eynollah: slope_deskew=slope_deskew, textline_light=self.textline_light, logger=self.logger,), - boxes, contours, contours_par, range(len(contours_par))) + boxes, contours, contours_par) results = list(results) # exhaust prior to release - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) + #textline_polygons, box_coord, slopes = zip(*results) self.logger.debug("exit get_slopes_and_deskew_new_light") return tuple(zip(*results)) def get_slopes_and_deskew_new(self, contours, contours_par, textline_mask_tot, boxes, slope_deskew): if not len(contours): - return [], [], [], [], [], [], [] + return [], [], [] self.logger.debug("enter get_slopes_and_deskew_new") with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: results = self.executor.map(partial(do_work_of_slopes_new, @@ -1924,16 +1920,16 @@ class Eynollah: KERNEL=KERNEL, logger=self.logger, plotter=self.plotter,), - boxes, contours, contours_par, range(len(contours_par))) + boxes, contours, contours_par) results = list(results) # exhaust prior to release - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) + #textline_polygons, box_coord, slopes = zip(*results) self.logger.debug("exit get_slopes_and_deskew_new") return tuple(zip(*results)) - def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, boxes, + def get_slopes_and_deskew_new_curved(self, contours_par, textline_mask_tot, boxes, mask_texts_only, num_col, scale_par, slope_deskew): - if not len(contours): - return [], [], [], [], [], [], [] + if not len(contours_par): + return [], [], [] self.logger.debug("enter get_slopes_and_deskew_new_curved") with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: with share_ndarray(mask_texts_only) as mask_texts_only_shared: @@ -1947,9 +1943,9 @@ class Eynollah: KERNEL=KERNEL, logger=self.logger, plotter=self.plotter,), - boxes, contours, contours_par, range(len(contours_par))) + boxes, contours_par) results = list(results) # exhaust prior to release - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) + #textline_polygons, box_coord, slopes = zip(*results) self.logger.debug("exit get_slopes_and_deskew_new_curved") return tuple(zip(*results)) @@ -4037,7 +4033,7 @@ class Eynollah: def filter_contours_without_textline_inside( - self, contours, text_con_org, contours_textline, + self, contours_par, contours_textline, contours_only_text_parent_d_ordered, conf_contours_textregions): @@ -4049,12 +4045,11 @@ class Eynollah: return [] return list(np.array(lis)[indices]) - return (filterfun(contours), - filterfun(text_con_org), - filterfun(conf_contours_textregions), + return (filterfun(contours_par), filterfun(contours_textline), filterfun(contours_only_text_parent_d_ordered), - indices + filterfun(conf_contours_textregions), + # indices ) def separate_marginals_to_left_and_right_and_order_from_top_to_down( @@ -4592,12 +4587,11 @@ class Eynollah: contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, marginal_cnts=polygons_of_marginals) #print("text region early 3.5 in %.1fs", time.time() - t0) - txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( + conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) - #txt_con_org = dilate_textregion_contours(txt_con_org) #contours_only_text_parent = dilate_textregion_contours(contours_only_text_parent) else: - txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( + conf_contours_textregions = get_textregion_contours_in_org_image_light( contours_only_text_parent, self.image, confidence_matrix) #print("text region early 4 in %.1fs", time.time() - t0) boxes_text = get_text_region_boxes_by_given_contours(contours_only_text_parent) @@ -4607,13 +4601,13 @@ class Eynollah: if not self.curved_line: if self.light_version: if self.textline_light: - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ - all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_light2( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea_org, + all_found_textline_polygons, \ + all_box_coord, slopes = self.get_slopes_and_deskew_new_light2( + contours_only_text_parent, textline_mask_tot_ea_org, boxes_text, slope_deskew) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_light2( - polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_org, + all_found_textline_polygons_marginals, \ + all_box_coord_marginals, slopes_marginals = self.get_slopes_and_deskew_new_light2( + polygons_of_marginals, textline_mask_tot_ea_org, boxes_marginals, slope_deskew) all_found_textline_polygons = dilate_textline_contours( @@ -4622,46 +4616,46 @@ class Eynollah: all_found_textline_polygons, None, textline_mask_tot_ea_org, type_contour="textline") all_found_textline_polygons_marginals = dilate_textline_contours( all_found_textline_polygons_marginals) - contours_only_text_parent, txt_con_org, conf_contours_textregions, \ - all_found_textline_polygons, contours_only_text_parent_d_ordered, \ - index_by_text_par_con = self.filter_contours_without_textline_inside( - contours_only_text_parent, txt_con_org, all_found_textline_polygons, + contours_only_text_parent, all_found_textline_polygons, \ + contours_only_text_parent_d_ordered, conf_contours_textregions = \ + self.filter_contours_without_textline_inside( + contours_only_text_parent, all_found_textline_polygons, contours_only_text_parent_d_ordered, conf_contours_textregions) else: textline_mask_tot_ea = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1) - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, \ - index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_light( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea, + all_found_textline_polygons, \ + all_box_coord, slopes = self.get_slopes_and_deskew_new_light( + contours_only_text_parent, contours_only_text_parent, textline_mask_tot_ea, boxes_text, slope_deskew) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_light( + all_found_textline_polygons_marginals, \ + all_box_coord_marginals, slopes_marginals = self.get_slopes_and_deskew_new_light( polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, boxes_marginals, slope_deskew) #all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( # all_found_textline_polygons, textline_mask_tot_ea_org, type_contour="textline") else: textline_mask_tot_ea = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1) - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ - all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea, + all_found_textline_polygons, \ + all_box_coord, slopes = self.get_slopes_and_deskew_new( + contours_only_text_parent, contours_only_text_parent, textline_mask_tot_ea, boxes_text, slope_deskew) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new( + all_found_textline_polygons_marginals, \ + all_box_coord_marginals, slopes_marginals = self.get_slopes_and_deskew_new( polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, boxes_marginals, slope_deskew) else: scale_param = 1 textline_mask_tot_ea_erode = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=2) - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ - all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_curved( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea_erode, + all_found_textline_polygons, \ + all_box_coord, slopes = self.get_slopes_and_deskew_new_curved( + contours_only_text_parent, textline_mask_tot_ea_erode, boxes_text, text_only, num_col_classifier, scale_param, slope_deskew) all_found_textline_polygons = small_textlines_to_parent_adherence2( all_found_textline_polygons, textline_mask_tot_ea, num_col_classifier) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_curved( - polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_erode, + all_found_textline_polygons_marginals, \ + all_box_coord_marginals, slopes_marginals = self.get_slopes_and_deskew_new_curved( + polygons_of_marginals, textline_mask_tot_ea_erode, boxes_marginals, text_only, num_col_classifier, scale_param, slope_deskew) all_found_textline_polygons_marginals = small_textlines_to_parent_adherence2( @@ -4884,7 +4878,7 @@ class Eynollah: conf_contours_textregions, conf_contours_textregions_h) else: pcgts = self.writer.build_pagexml_no_full_layout( - txt_con_org, page_coord, order_text_new, id_of_texts_tot, + contours_only_text_parent, page_coord, order_text_new, id_of_texts_tot, all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals_left, polygons_of_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index fb4bbd0..2560846 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -216,7 +216,7 @@ def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first def get_textregion_contours_in_org_image_light(cnts, img, confidence_matrix): if not len(cnts): - return [], [] + return [] confidence_matrix = cv2.resize(confidence_matrix, (img.shape[1] // 6, img.shape[0] // 6), @@ -226,7 +226,7 @@ def get_textregion_contours_in_org_image_light(cnts, img, confidence_matrix): cnt_mask = np.zeros(confidence_matrix.shape) cnt_mask = cv2.fillPoly(cnt_mask, pts=[cnt // 6], color=1.0) confs.append(np.sum(confidence_matrix * cnt_mask) / np.sum(cnt_mask)) - return cnts, confs + return confs def return_contours_of_interested_textline(region_pre_p, label): # pixels of images are identified by 5 diff --git a/src/eynollah/utils/separate_lines.py b/src/eynollah/utils/separate_lines.py index 3bfc903..22ef00d 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/src/eynollah/utils/separate_lines.py @@ -1592,7 +1592,7 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map @wrap_ndarray_shared(kw='textline_mask_tot_ea') def do_work_of_slopes_new( - box_text, contour, contour_par, index_r_con, + box_text, contour, contour_par, textline_mask_tot_ea=None, slope_deskew=0.0, logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None ): @@ -1647,12 +1647,12 @@ def do_work_of_slopes_new( all_text_region_raw[mask_only_con_region == 0] = 0 cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text) - return cnt_clean_rot, box_text, contour, contour_par, crop_coor, index_r_con, slope + return cnt_clean_rot, crop_coor, slope @wrap_ndarray_shared(kw='textline_mask_tot_ea') @wrap_ndarray_shared(kw='mask_texts_only') def do_work_of_slopes_new_curved( - box_text, contour, contour_par, index_r_con, + box_text, contour_par, textline_mask_tot_ea=None, mask_texts_only=None, num_col=1, scale_par=1.0, slope_deskew=0.0, logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None @@ -1743,11 +1743,11 @@ def do_work_of_slopes_new_curved( slope_for_all, contour_par, box_text, True) - return textlines_cnt_per_region[::-1], box_text, contour, contour_par, crop_coor, index_r_con, slope + return textlines_cnt_per_region[::-1], crop_coor, slope @wrap_ndarray_shared(kw='textline_mask_tot_ea') def do_work_of_slopes_new_light( - box_text, contour, contour_par, index_r_con, + box_text, contour, contour_par, textline_mask_tot_ea=None, slope_deskew=0, textline_light=True, logger=None ): @@ -1777,4 +1777,4 @@ def do_work_of_slopes_new_light( all_text_region_raw[mask_only_con_region == 0] = 0 cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_deskew, contour_par, box_text) - return cnt_clean_rot, box_text, contour, contour_par, crop_coor, index_r_con, slope_deskew + return cnt_clean_rot, crop_coor, slope_deskew From e32479765cc52a29462b36f876d253478860f176 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 7 Oct 2025 23:03:27 +0200 Subject: [PATCH 342/374] writer: simplify - simplify serialization of coordinates - re-use `serialize_lines_in_region` (drop `*_in_dropcapital` and `*_in_marginal`) - re-use `calculate_polygon_coords` --- src/eynollah/writer.py | 343 ++++++++++++++++------------------------- 1 file changed, 131 insertions(+), 212 deletions(-) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 936c95f..67a2989 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -56,113 +56,30 @@ class EynollahXmlWriter(): points_page_print = points_page_print + ' ' return points_page_print[:-1] - def serialize_lines_in_marginal(self, marginal_region, all_found_textline_polygons_marginals, marginal_idx, page_coord, all_box_coord_marginals, slopes_marginals, counter, ocr_all_textlines_textregion): - for j in range(len(all_found_textline_polygons_marginals[marginal_idx])): - coords = CoordsType() - textline = TextLineType(id=counter.next_line_id, Coords=coords) - if ocr_all_textlines_textregion: - textline.set_TextEquiv( [ TextEquivType(Unicode=ocr_all_textlines_textregion[j]) ] ) - marginal_region.add_TextLine(textline) - marginal_region.set_orientation(-slopes_marginals[marginal_idx]) - points_co = '' - for l in range(len(all_found_textline_polygons_marginals[marginal_idx][j])): - if not (self.curved_line or self.textline_light): - if len(all_found_textline_polygons_marginals[marginal_idx][j][l]) == 2: - textline_x_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x) ) - textline_y_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y) ) - else: - textline_x_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x) ) - textline_y_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y) ) - points_co += str(textline_x_coord) - points_co += ',' - points_co += str(textline_y_coord) - if (self.curved_line or self.textline_light) and np.abs(slopes_marginals[marginal_idx]) <= 45: - if len(all_found_textline_polygons_marginals[marginal_idx][j][l]) == 2: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][1] + page_coord[0]) / self.scale_y)) - else: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][1] + page_coord[0]) / self.scale_y)) - - elif (self.curved_line or self.textline_light) and np.abs(slopes_marginals[marginal_idx]) > 45: - if len(all_found_textline_polygons_marginals[marginal_idx][j][l]) == 2: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y)) - else: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y)) - points_co += ' ' - coords.set_points(points_co[:-1]) - def serialize_lines_in_region(self, text_region, all_found_textline_polygons, region_idx, page_coord, all_box_coord, slopes, counter, ocr_all_textlines_textregion): self.logger.debug('enter serialize_lines_in_region') - for j in range(len(all_found_textline_polygons[region_idx])): + for j, polygon_textline in enumerate(all_found_textline_polygons[region_idx]): coords = CoordsType() textline = TextLineType(id=counter.next_line_id, Coords=coords) if ocr_all_textlines_textregion: - textline.set_TextEquiv( [ TextEquivType(Unicode=ocr_all_textlines_textregion[j]) ] ) + # FIXME: add OCR confidence + textline.set_TextEquiv([TextEquivType(Unicode=ocr_all_textlines_textregion[j])]) text_region.add_TextLine(textline) text_region.set_orientation(-slopes[region_idx]) region_bboxes = all_box_coord[region_idx] points_co = '' - for idx_contour_textline, contour_textline in enumerate(all_found_textline_polygons[region_idx][j]): - if not (self.curved_line or self.textline_light): - if len(contour_textline) == 2: - textline_x_coord = max(0, int((contour_textline[0] + region_bboxes[2] + page_coord[2]) / self.scale_x)) - textline_y_coord = max(0, int((contour_textline[1] + region_bboxes[0] + page_coord[0]) / self.scale_y)) - else: - textline_x_coord = max(0, int((contour_textline[0][0] + region_bboxes[2] + page_coord[2]) / self.scale_x)) - textline_y_coord = max(0, int((contour_textline[0][1] + region_bboxes[0] + page_coord[0]) / self.scale_y)) - points_co += str(textline_x_coord) - points_co += ',' - points_co += str(textline_y_coord) - - if self.textline_light or (self.curved_line and np.abs(slopes[region_idx]) <= 45): - if len(contour_textline) == 2: - points_co += str(int((contour_textline[0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[1] + page_coord[0]) / self.scale_y)) - else: - points_co += str(int((contour_textline[0][0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[0][1] + page_coord[0])/self.scale_y)) - elif self.curved_line and np.abs(slopes[region_idx]) > 45: - if len(contour_textline)==2: - points_co += str(int((contour_textline[0] + region_bboxes[2] + page_coord[2])/self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[1] + region_bboxes[0] + page_coord[0])/self.scale_y)) - else: - points_co += str(int((contour_textline[0][0] + region_bboxes[2]+page_coord[2])/self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[0][1] + region_bboxes[0]+page_coord[0])/self.scale_y)) - points_co += ' ' - coords.set_points(points_co[:-1]) - - def serialize_lines_in_dropcapital(self, text_region, all_found_textline_polygons, region_idx, page_coord, all_box_coord, slopes, counter, ocr_all_textlines_textregion): - self.logger.debug('enter serialize_lines_in_region') - for j in range(1): - coords = CoordsType() - textline = TextLineType(id=counter.next_line_id, Coords=coords) - if ocr_all_textlines_textregion: - textline.set_TextEquiv( [ TextEquivType(Unicode=ocr_all_textlines_textregion[j]) ] ) - text_region.add_TextLine(textline) - #region_bboxes = all_box_coord[region_idx] - points_co = '' - for idx_contour_textline, contour_textline in enumerate(all_found_textline_polygons[j]): - if len(contour_textline) == 2: - points_co += str(int((contour_textline[0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[1] + page_coord[0]) / self.scale_y)) - else: - points_co += str(int((contour_textline[0][0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[0][1] + page_coord[0])/self.scale_y)) - - points_co += ' ' + for point in polygon_textline: + if len(point) != 2: + point = point[0] + point_x = point[0] + page_coord[2] + point_y = point[1] + page_coord[0] + # FIXME: or actually... not self.textline_light and not self.curved_line or np.abs(slopes[region_idx]) > 45? + if not self.textline_light and not (self.curved_line and np.abs(slopes[region_idx]) <= 45): + point_x += region_bboxes[2] + point_y += region_bboxes[0] + point_x = max(0, int(point_x / self.scale_x)) + point_y = max(0, int(point_y / self.scale_y)) + points_co += str(point_x) + ',' + str(point_y) + ' ' coords.set_points(points_co[:-1]) def write_pagexml(self, pcgts): @@ -170,7 +87,7 @@ class EynollahXmlWriter(): with open(self.output_filename, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals_left, found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, conf_contours_textregion=None, skip_layout_reading_order=False): + def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals_left, found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_seplines, found_polygons_tables, ocr_all_textlines=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, conf_contours_textregion=None, skip_layout_reading_order=False): self.logger.debug('enter build_pagexml_no_full_layout') # create the file structure @@ -179,90 +96,79 @@ class EynollahXmlWriter(): page.set_Border(BorderType(Coords=CoordsType(points=self.calculate_page_coords(cont_page)))) counter = EynollahIdCounter() - if len(found_polygons_text_region) > 0: + if len(order_of_texts): _counter_marginals = EynollahIdCounter(region_idx=len(order_of_texts)) - id_of_marginalia_left = [_counter_marginals.next_region_id for _ in found_polygons_marginals_left] - id_of_marginalia_right = [_counter_marginals.next_region_id for _ in found_polygons_marginals_right] + id_of_marginalia_left = [_counter_marginals.next_region_id + for _ in found_polygons_marginals_left] + id_of_marginalia_right = [_counter_marginals.next_region_id + for _ in found_polygons_marginals_right] xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right) - for mm in range(len(found_polygons_text_region)): - textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord, skip_layout_reading_order), conf=conf_contours_textregion[mm]), - ) - #textregion.set_conf(conf_contours_textregion[mm]) + for mm, region_contour in enumerate(found_polygons_text_region): + textregion = TextRegionType( + id=counter.next_region_id, type_='paragraph', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord, + skip_layout_reading_order), + conf=conf_contours_textregion[mm]), + ) page.add_TextRegion(textregion) if ocr_all_textlines: ocr_textlines = ocr_all_textlines[mm] else: ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, all_box_coord, slopes, counter, ocr_textlines) + self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, + all_box_coord, slopes, counter, ocr_textlines) - for mm in range(len(found_polygons_marginals_left)): - marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_left[mm], page_coord))) + for mm, region_contour in enumerate(found_polygons_marginals_left): + marginal = TextRegionType( + id=counter.next_region_id, type_='marginalia', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_TextRegion(marginal) if ocr_all_textlines_marginals_left: ocr_textlines = ocr_all_textlines_marginals_left[mm] else: ocr_textlines = None - - #print(ocr_textlines, mm, len(all_found_textline_polygons_marginals_left[mm]) ) - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) + self.serialize_lines_in_region(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, + all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) - for mm in range(len(found_polygons_marginals_right)): - marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_right[mm], page_coord))) + for mm, region_contour in enumerate(found_polygons_marginals_right): + marginal = TextRegionType( + id=counter.next_region_id, type_='marginalia', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_TextRegion(marginal) if ocr_all_textlines_marginals_right: ocr_textlines = ocr_all_textlines_marginals_right[mm] else: ocr_textlines = None - - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) + self.serialize_lines_in_region(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, + all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) - for mm in range(len(found_polygons_text_region_img)): - img_region = ImageRegionType(id=counter.next_region_id, Coords=CoordsType()) + for region_contour in found_polygons_text_region_img: + img_region = ImageRegionType( + id=counter.next_region_id, + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_ImageRegion(img_region) - points_co = '' - for lmm in range(len(found_polygons_text_region_img[mm])): - try: - points_co += str(int((found_polygons_text_region_img[mm][lmm,0,0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((found_polygons_text_region_img[mm][lmm,0,1] + page_coord[0]) / self.scale_y)) - points_co += ' ' - except: - points_co += str(int((found_polygons_text_region_img[mm][lmm][0] + page_coord[2])/ self.scale_x )) - points_co += ',' - points_co += str(int((found_polygons_text_region_img[mm][lmm][1] + page_coord[0])/ self.scale_y )) - points_co += ' ' + for region_contour in polygons_seplines: + sep = SeparatorRegionType( + id=counter.next_region_id, + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, [0, 0, 0, 0])) + ) + page.add_SeparatorRegion(sep) - img_region.get_Coords().set_points(points_co[:-1]) - - for mm in range(len(polygons_lines_to_be_written_in_xml)): - sep_hor = SeparatorRegionType(id=counter.next_region_id, Coords=CoordsType()) - page.add_SeparatorRegion(sep_hor) - points_co = '' - for lmm in range(len(polygons_lines_to_be_written_in_xml[mm])): - points_co += str(int((polygons_lines_to_be_written_in_xml[mm][lmm,0,0] ) / self.scale_x)) - points_co += ',' - points_co += str(int((polygons_lines_to_be_written_in_xml[mm][lmm,0,1] ) / self.scale_y)) - points_co += ' ' - sep_hor.get_Coords().set_points(points_co[:-1]) - for mm in range(len(found_polygons_tables)): - tab_region = TableRegionType(id=counter.next_region_id, Coords=CoordsType()) - page.add_TableRegion(tab_region) - points_co = '' - for lmm in range(len(found_polygons_tables[mm])): - points_co += str(int((found_polygons_tables[mm][lmm,0,0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((found_polygons_tables[mm][lmm,0,1] + page_coord[0]) / self.scale_y)) - points_co += ' ' - tab_region.get_Coords().set_points(points_co[:-1]) + for region_contour in found_polygons_tables: + tab = TableRegionType( + id=counter.next_region_id, + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) + page.add_TableRegion(tab) return pcgts - def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals_left,found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_lines_to_be_written_in_xml, ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, ocr_all_textlines_drop=None, conf_contours_textregion=None, conf_contours_textregion_h=None): + def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals_left,found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_seplines, ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, ocr_all_textlines_drop=None, conf_contours_textregion=None, conf_contours_textregion_h=None): self.logger.debug('enter build_pagexml_full_layout') # create the file structure @@ -271,99 +177,112 @@ class EynollahXmlWriter(): page.set_Border(BorderType(Coords=CoordsType(points=self.calculate_page_coords(cont_page)))) counter = EynollahIdCounter() - _counter_marginals = EynollahIdCounter(region_idx=len(order_of_texts)) - id_of_marginalia_left = [_counter_marginals.next_region_id for _ in found_polygons_marginals_left] - id_of_marginalia_right = [_counter_marginals.next_region_id for _ in found_polygons_marginals_right] - xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right) + if len(order_of_texts): + _counter_marginals = EynollahIdCounter(region_idx=len(order_of_texts)) + id_of_marginalia_left = [_counter_marginals.next_region_id + for _ in found_polygons_marginals_left] + id_of_marginalia_right = [_counter_marginals.next_region_id + for _ in found_polygons_marginals_right] + xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right) - for mm in range(len(found_polygons_text_region)): - textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord), conf=conf_contours_textregion[mm])) + for mm, region_contour in enumerate(found_polygons_text_region): + textregion = TextRegionType( + id=counter.next_region_id, type_='paragraph', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord), + conf=conf_contours_textregion[mm]) + ) page.add_TextRegion(textregion) - if ocr_all_textlines: ocr_textlines = ocr_all_textlines[mm] else: ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, all_box_coord, slopes, counter, ocr_textlines) + self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, + all_box_coord, slopes, counter, ocr_textlines) self.logger.debug('len(found_polygons_text_region_h) %s', len(found_polygons_text_region_h)) - for mm in range(len(found_polygons_text_region_h)): - textregion = TextRegionType(id=counter.next_region_id, type_='heading', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_h[mm], page_coord))) + for mm, region_contour in enumerate(found_polygons_text_region_h): + textregion = TextRegionType( + id=counter.next_region_id, type_='heading', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_TextRegion(textregion) - if ocr_all_textlines_h: ocr_textlines = ocr_all_textlines_h[mm] else: ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons_h, mm, page_coord, all_box_coord_h, slopes_h, counter, ocr_textlines) + self.serialize_lines_in_region(textregion, all_found_textline_polygons_h, mm, page_coord, + all_box_coord_h, slopes_h, counter, ocr_textlines) - for mm in range(len(found_polygons_marginals_left)): - marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_left[mm], page_coord))) + for mm, region_contour in enumerate(found_polygons_marginals_left): + marginal = TextRegionType( + id=counter.next_region_id, type_='marginalia', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_TextRegion(marginal) if ocr_all_textlines_marginals_left: ocr_textlines = ocr_all_textlines_marginals_left[mm] else: ocr_textlines = None - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) - - for mm in range(len(found_polygons_marginals_right)): - marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals_right[mm], page_coord))) + self.serialize_lines_in_region(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) + + for mm, region_contour in enumerate(found_polygons_marginals_right): + marginal = TextRegionType( + id=counter.next_region_id, type_='marginalia', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_TextRegion(marginal) if ocr_all_textlines_marginals_right: ocr_textlines = ocr_all_textlines_marginals_right[mm] else: ocr_textlines = None - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) - - for mm in range(len(found_polygons_drop_capitals)): - dropcapital = TextRegionType(id=counter.next_region_id, type_='drop-capital', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_drop_capitals[mm], page_coord))) + self.serialize_lines_in_region(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, + all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) + + for mm, region_contour in enumerate(found_polygons_drop_capitals): + dropcapital = TextRegionType( + id=counter.next_region_id, type_='drop-capital', + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) + ) page.add_TextRegion(dropcapital) - all_box_coord_drop = None - slopes_drop = None + all_box_coord_drop = [[0, 0, 0, 0]] + slopes_drop = [0] if ocr_all_textlines_drop: ocr_textlines = ocr_all_textlines_drop[mm] else: ocr_textlines = None - self.serialize_lines_in_dropcapital(dropcapital, [found_polygons_drop_capitals[mm]], mm, page_coord, all_box_coord_drop, slopes_drop, counter, ocr_all_textlines_textregion=ocr_textlines) + self.serialize_lines_in_region(dropcapital, [[found_polygons_drop_capitals[mm]]], 0, page_coord, + all_box_coord_drop, slopes_drop, counter, ocr_textlines) - for mm in range(len(found_polygons_text_region_img)): - page.add_ImageRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_img[mm], page_coord)))) + for region_contour in found_polygons_text_region_img: + page.add_ImageRegion( + ImageRegionType(id=counter.next_region_id, + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)))) - for mm in range(len(polygons_lines_to_be_written_in_xml)): - page.add_SeparatorRegion(SeparatorRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(polygons_lines_to_be_written_in_xml[mm], [0 , 0, 0, 0])))) + for region_contour in polygons_seplines: + page.add_SeparatorRegion( + SeparatorRegionType(id=counter.next_region_id, + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, [0, 0, 0, 0])))) - for mm in range(len(found_polygons_tables)): - page.add_TableRegion(TableRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_tables[mm], page_coord)))) + for region_contour in found_polygons_tables: + page.add_TableRegion( + TableRegionType(id=counter.next_region_id, + Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)))) return pcgts def calculate_polygon_coords(self, contour, page_coord, skip_layout_reading_order=False): self.logger.debug('enter calculate_polygon_coords') coords = '' - for value_bbox in contour: - if skip_layout_reading_order: - if len(value_bbox) == 2: - coords += str(int((value_bbox[0]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[1]) / self.scale_y)) - else: - coords += str(int((value_bbox[0][0]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[0][1]) / self.scale_y)) - else: - if len(value_bbox) == 2: - coords += str(int((value_bbox[0] + page_coord[2]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[1] + page_coord[0]) / self.scale_y)) - else: - coords += str(int((value_bbox[0][0] + page_coord[2]) / self.scale_x)) - coords += ',' - coords += str(int((value_bbox[0][1] + page_coord[0]) / self.scale_y)) - coords=coords + ' ' + for point in contour: + if len(point) != 2: + point = point[0] + point_x = point[0] + point_y = point[1] + if not skip_layout_reading_order: + point_x += page_coord[2] + point_y += page_coord[0] + point_x = int(point_x / self.scale_x) + point_y = int(point_y / self.scale_y) + coords += str(point_x) + ',' + str(point_y) + ' ' return coords[:-1] From cbbb3248c72c1f3e50b98de1f7e2980bdd14da5d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 00:43:29 +0200 Subject: [PATCH 343/374] writer: simplify - `build_pagexml_no_full_layout`: delegate to `build_pagexml_full_layout` (removing redundant code) --- src/eynollah/writer.py | 133 +++++++++++++++-------------------------- 1 file changed, 49 insertions(+), 84 deletions(-) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index 67a2989..eee7440 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -87,8 +87,50 @@ class EynollahXmlWriter(): with open(self.output_filename, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals_left, found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_seplines, found_polygons_tables, ocr_all_textlines=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, conf_contours_textregion=None, skip_layout_reading_order=False): - self.logger.debug('enter build_pagexml_no_full_layout') + def build_pagexml_no_full_layout( + self, found_polygons_text_region, + page_coord, order_of_texts, id_of_texts, + all_found_textline_polygons, + all_box_coord, + found_polygons_text_region_img, + found_polygons_marginals_left, found_polygons_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, + found_polygons_tables, + **kwargs): + return self.build_pagexml_full_layout( + found_polygons_text_region, [], + page_coord, order_of_texts, id_of_texts, + all_found_textline_polygons, [], + all_box_coord, [], + found_polygons_text_region_img, found_polygons_tables, [], + found_polygons_marginals_left, found_polygons_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, [], slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, + **kwargs) + + def build_pagexml_full_layout( + self, + found_polygons_text_region, found_polygons_text_region_h, + page_coord, order_of_texts, id_of_texts, + all_found_textline_polygons, all_found_textline_polygons_h, + all_box_coord, all_box_coord_h, + found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, + found_polygons_marginals_left,found_polygons_marginals_right, + all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, + all_box_coord_marginals_left, all_box_coord_marginals_right, + slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, + cont_page, polygons_seplines, + ocr_all_textlines=None, ocr_all_textlines_h=None, + ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, + ocr_all_textlines_drop=None, + conf_contours_textregion=None, conf_contours_textregion_h=None, + skip_layout_reading_order=False): + self.logger.debug('enter build_pagexml') # create the file structure pcgts = self.pcgts if self.pcgts else create_page_xml(self.image_filename, self.height_org, self.width_org) @@ -108,89 +150,10 @@ class EynollahXmlWriter(): textregion = TextRegionType( id=counter.next_region_id, type_='paragraph', Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord, - skip_layout_reading_order), - conf=conf_contours_textregion[mm]), - ) - page.add_TextRegion(textregion) - if ocr_all_textlines: - ocr_textlines = ocr_all_textlines[mm] - else: - ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, - all_box_coord, slopes, counter, ocr_textlines) - - for mm, region_contour in enumerate(found_polygons_marginals_left): - marginal = TextRegionType( - id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) - ) - page.add_TextRegion(marginal) - if ocr_all_textlines_marginals_left: - ocr_textlines = ocr_all_textlines_marginals_left[mm] - else: - ocr_textlines = None - self.serialize_lines_in_region(marginal, all_found_textline_polygons_marginals_left, mm, page_coord, - all_box_coord_marginals_left, slopes_marginals_left, counter, ocr_textlines) - - for mm, region_contour in enumerate(found_polygons_marginals_right): - marginal = TextRegionType( - id=counter.next_region_id, type_='marginalia', - Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) - ) - page.add_TextRegion(marginal) - if ocr_all_textlines_marginals_right: - ocr_textlines = ocr_all_textlines_marginals_right[mm] - else: - ocr_textlines = None - self.serialize_lines_in_region(marginal, all_found_textline_polygons_marginals_right, mm, page_coord, - all_box_coord_marginals_right, slopes_marginals_right, counter, ocr_textlines) - - for region_contour in found_polygons_text_region_img: - img_region = ImageRegionType( - id=counter.next_region_id, - Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) - ) - page.add_ImageRegion(img_region) - - for region_contour in polygons_seplines: - sep = SeparatorRegionType( - id=counter.next_region_id, - Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, [0, 0, 0, 0])) - ) - page.add_SeparatorRegion(sep) - - for region_contour in found_polygons_tables: - tab = TableRegionType( - id=counter.next_region_id, - Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) - ) - page.add_TableRegion(tab) - - return pcgts - - def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals_left,found_polygons_marginals_right, all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_h, slopes_marginals_left, slopes_marginals_right, cont_page, polygons_seplines, ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, ocr_all_textlines_drop=None, conf_contours_textregion=None, conf_contours_textregion_h=None): - self.logger.debug('enter build_pagexml_full_layout') - - # create the file structure - pcgts = self.pcgts if self.pcgts else create_page_xml(self.image_filename, self.height_org, self.width_org) - page = pcgts.get_Page() - page.set_Border(BorderType(Coords=CoordsType(points=self.calculate_page_coords(cont_page)))) - - counter = EynollahIdCounter() - if len(order_of_texts): - _counter_marginals = EynollahIdCounter(region_idx=len(order_of_texts)) - id_of_marginalia_left = [_counter_marginals.next_region_id - for _ in found_polygons_marginals_left] - id_of_marginalia_right = [_counter_marginals.next_region_id - for _ in found_polygons_marginals_right] - xml_reading_order(page, order_of_texts, id_of_marginalia_left, id_of_marginalia_right) - - for mm, region_contour in enumerate(found_polygons_text_region): - textregion = TextRegionType( - id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord), - conf=conf_contours_textregion[mm]) + skip_layout_reading_order)) ) + if conf_contours_textregion: + textregion.Coords.set_conf(conf_contours_textregion[mm]) page.add_TextRegion(textregion) if ocr_all_textlines: ocr_textlines = ocr_all_textlines[mm] @@ -205,6 +168,8 @@ class EynollahXmlWriter(): id=counter.next_region_id, type_='heading', Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) ) + if conf_contours_textregion_h: + textregion.Coords.set_conf(conf_contours_textregion_h[mm]) page.add_TextRegion(textregion) if ocr_all_textlines_h: ocr_textlines = ocr_all_textlines_h[mm] From 75823f9bed64153718acab6f664cdfc114ef34fb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 00:54:53 +0200 Subject: [PATCH 344/374] run_single: call `writer.build_pagexml_no_full_layout` w/ kwargs --- src/eynollah/eynollah.py | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index ec68bcd..b109c90 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4260,18 +4260,6 @@ class Eynollah: order_text_new = [0] slopes =[0] id_of_texts_tot =['region_0001'] - - polygons_of_images = [] - slopes_marginals_left = [] - slopes_marginals_right = [] - polygons_of_marginals_left = [] - polygons_of_marginals_right = [] - all_found_textline_polygons_marginals_left = [] - all_found_textline_polygons_marginals_right = [] - all_box_coord_marginals_left = [] - all_box_coord_marginals_right = [] - polygons_seplines = [] - contours_tables = [] conf_contours_textregions =[0] if self.ocr and not self.tr: @@ -4284,15 +4272,13 @@ class Eynollah: pcgts = self.writer.build_pagexml_no_full_layout( cont_page, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, page_coord, polygons_of_images, - polygons_of_marginals_left, polygons_of_marginals_right, - all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, - all_box_coord_marginals_left, all_box_coord_marginals_right, - slopes, slopes_marginals_left, slopes_marginals_right, - cont_page, polygons_seplines, contours_tables, + all_found_textline_polygons, page_coord, [], + [], [], [], [], [], [], + slopes, [], [], + cont_page, [], [], ocr_all_textlines=ocr_all_textlines, conf_contours_textregion=conf_contours_textregions, - skip_layout_reading_order=self.skip_layout_and_reading_order) + skip_layout_reading_order=True) self.logger.info("Basic processing complete") return pcgts @@ -4884,9 +4870,11 @@ class Eynollah: all_found_textline_polygons_marginals_left, all_found_textline_polygons_marginals_right, all_box_coord_marginals_left, all_box_coord_marginals_right, slopes, slopes_marginals_left, slopes_marginals_right, - cont_page, polygons_seplines, contours_tables, ocr_all_textlines, - ocr_all_textlines_marginals_left, ocr_all_textlines_marginals_right, - conf_contours_textregions) + cont_page, polygons_seplines, contours_tables, + ocr_all_textlines=ocr_all_textlines, + ocr_all_textlines_marginals_left=ocr_all_textlines_marginals_left, + ocr_all_textlines_marginals_right=ocr_all_textlines_marginals_right, + conf_contours_textregions=conf_contours_textregions) return pcgts From 5e11a68a3e18e926b25829e0fce3c279e529aca0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 01:03:48 +0200 Subject: [PATCH 345/374] writer/run_single: consistent kwarg naming `conf_contours_textregion(s)` --- src/eynollah/writer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/eynollah/writer.py b/src/eynollah/writer.py index eee7440..8859d95 100644 --- a/src/eynollah/writer.py +++ b/src/eynollah/writer.py @@ -128,7 +128,7 @@ class EynollahXmlWriter(): ocr_all_textlines=None, ocr_all_textlines_h=None, ocr_all_textlines_marginals_left=None, ocr_all_textlines_marginals_right=None, ocr_all_textlines_drop=None, - conf_contours_textregion=None, conf_contours_textregion_h=None, + conf_contours_textregions=None, conf_contours_textregions_h=None, skip_layout_reading_order=False): self.logger.debug('enter build_pagexml') @@ -152,8 +152,8 @@ class EynollahXmlWriter(): Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord, skip_layout_reading_order)) ) - if conf_contours_textregion: - textregion.Coords.set_conf(conf_contours_textregion[mm]) + if conf_contours_textregions: + textregion.Coords.set_conf(conf_contours_textregions[mm]) page.add_TextRegion(textregion) if ocr_all_textlines: ocr_textlines = ocr_all_textlines[mm] @@ -168,8 +168,8 @@ class EynollahXmlWriter(): id=counter.next_region_id, type_='heading', Coords=CoordsType(points=self.calculate_polygon_coords(region_contour, page_coord)) ) - if conf_contours_textregion_h: - textregion.Coords.set_conf(conf_contours_textregion_h[mm]) + if conf_contours_textregions_h: + textregion.Coords.set_conf(conf_contours_textregions_h[mm]) page.add_TextRegion(textregion) if ocr_all_textlines_h: ocr_textlines = ocr_all_textlines_h[mm] From ca72a095cab373b6daa2f7353f456d9eacfd399b Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 00:44:32 +0200 Subject: [PATCH 346/374] tests: cover table detection in various modes --- tests/test_run.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/test_run.py b/tests/test_run.py index 98cee30..79c64c2 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -67,6 +67,44 @@ def test_run_eynollah_layout_filename(tmp_path, pytestconfig, caplog, options): lines = tree.xpath("//page:TextLine", namespaces=NS) assert len(lines) == 31, "result is inaccurate" # 29 paragraph lines, 1 page and 1 catch-word line +@pytest.mark.parametrize( + "options", + [ + ["--tables"], + ["--tables", "--full-layout"], + ["--tables", "--full-layout", "--textline_light", "--light_version"], + ], ids=str) +def test_run_eynollah_layout_filename2(tmp_path, pytestconfig, caplog, options): + infile = testdir.joinpath('resources/euler_rechenkunst01_1738_0025.tif') + outfile = tmp_path / 'euler_rechenkunst01_1738_0025.xml' + args = [ + '-m', MODELS_LAYOUT, + '-i', str(infile), + '-o', str(outfile.parent), + ] + if pytestconfig.getoption('verbose') > 0: + args.extend(['-l', 'DEBUG']) + caplog.set_level(logging.INFO) + def only_eynollah(logrec): + return logrec.name == 'eynollah' + runner = CliRunner() + with caplog.filtering(only_eynollah): + result = runner.invoke(layout_cli, args + options, catch_exceptions=False) + assert result.exit_code == 0, result.stdout + logmsgs = [logrec.message for logrec in caplog.records] + assert str(infile) in logmsgs + assert outfile.exists() + tree = page_from_file(str(outfile)).etree + regions = tree.xpath("//page:TextRegion", namespaces=NS) + assert len(regions) >= 2, "result is inaccurate" + regions = tree.xpath("//page:TableRegion", namespaces=NS) + # model/decoding is not very precise, so (depending on mode) we can get fractures/splits/FP + assert len(regions) >= 1, "result is inaccurate" + regions = tree.xpath("//page:SeparatorRegion", namespaces=NS) + assert len(regions) >= 2, "result is inaccurate" + lines = tree.xpath("//page:TextLine", namespaces=NS) + assert len(lines) >= 2, "result is inaccurate" # mostly table (if detected correctly), but 1 page and 1 catch-word line + def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): indir = testdir.joinpath('resources') outdir = tmp_path From e5b52645685b669d5af7c5da2870a01660f81cdb Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 12:17:53 +0200 Subject: [PATCH 347/374] CI: add diagnostic message for model symlink --- .github/workflows/test-eynollah.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index 7c3f5ae..759b26c 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -66,6 +66,7 @@ jobs: python -m pip install --upgrade pip make install-dev EXTRAS=OCR,plotting make deps-test EXTRAS=OCR,plotting + ls -l models_* - name: Test with pytest run: make coverage PYTEST_ARGS="-vv --junitxml=pytest.xml" - name: Get coverage results From 839b7c4d846d6f73069529aa1f337caa362917c0 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 12:33:14 +0200 Subject: [PATCH 348/374] make models: avoid re-download --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 618b1f9..29dd877 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,9 @@ help: # Download and extract models to $(PWD)/models_layout_v0_5_0 models: $(BIN_MODELNAME) $(SEG_MODELNAME) $(OCR_MODELNAME) +# do not download these files if we already have the directories +.INTERMEDIATE: $(BIN_MODELFILE) $(SEG_MODELFILE) $(OCR_MODELFILE) + $(BIN_MODELFILE): wget -O $@ $(BIN_MODEL) $(SEG_MODELFILE): From 1d4815b48f1f5b1bf006efe78141fd3161ee8073 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 14:56:14 +0200 Subject: [PATCH 349/374] utils_ocr: forgot to pass coordinate offsets --- src/eynollah/eynollah.py | 24 ++++++++++++------------ src/eynollah/utils/utils_ocr.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index b109c90..a6b65c4 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4265,8 +4265,8 @@ class Eynollah: if self.ocr and not self.tr: gc.collect() ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons, self.prediction_model, - self.b_s_ocr, self.num_to_char, textline_light=True) + image_page, all_found_textline_polygons, np.zeros((len(all_found_textline_polygons), 4)), + self.prediction_model, self.b_s_ocr, self.num_to_char, textline_light=True) else: ocr_all_textlines = None @@ -4756,36 +4756,36 @@ class Eynollah: if len(all_found_textline_polygons)>0: ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + image_page, all_found_textline_polygons, all_box_coord, + self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines = None if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons_marginals_left, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + image_page, all_found_textline_polygons_marginals_left, all_box_coord_marginals_left, + self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_marginals_left = None if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons_marginals_right, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + image_page, all_found_textline_polygons_marginals_right, all_box_coord_marginals_right, + self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_marginals_right = None if all_found_textline_polygons_h and len(all_found_textline_polygons)>0: ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines( - image_page, all_found_textline_polygons_h, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + image_page, all_found_textline_polygons_h, all_box_coord_h, + self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_h = None if polygons_of_drop_capitals and len(polygons_of_drop_capitals)>0: ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines( - image_page, polygons_of_drop_capitals, self.prediction_model, - self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + image_page, polygons_of_drop_capitals, np.zeros((len(polygons_of_drop_capitals), 4)), + self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: ocr_all_textlines_drop = None diff --git a/src/eynollah/utils/utils_ocr.py b/src/eynollah/utils/utils_ocr.py index 602ad6e..6e71b0f 100644 --- a/src/eynollah/utils/utils_ocr.py +++ b/src/eynollah/utils/utils_ocr.py @@ -1,13 +1,17 @@ +import math +import copy + import numpy as np import cv2 import tensorflow as tf from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d -import math from PIL import Image, ImageDraw, ImageFont from Bio import pairwise2 + from .resize import resize_image + def decode_batch_predictions(pred, num_to_char, max_len = 128): # input_len is the product of the batch size and the # number of time steps. @@ -370,7 +374,9 @@ def return_textline_contour_with_added_box_coordinate(textline_contour, box_ind return textline_contour -def return_rnn_cnn_ocr_of_given_textlines(image, all_found_textline_polygons, +def return_rnn_cnn_ocr_of_given_textlines(image, + all_found_textline_polygons, + all_box_coord, prediction_model, b_s_ocr, num_to_char, textline_light=False, From 027b87d32125afdc1bebbb968fc32b55b58bf153 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 14:56:57 +0200 Subject: [PATCH 350/374] fixup c0137c2 (missing arguments for utils_ocr) --- src/eynollah/eynollah.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index a6b65c4..aeb01be 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -96,6 +96,7 @@ from .utils.rotate import ( rotation_image_new ) from .utils.utils_ocr import ( + return_start_and_end_of_common_text_of_textline_ocr_without_common_section, return_textline_contour_with_added_box_coordinate, preprocess_and_resize_image_for_ocrcnn_model, return_textlines_split_if_needed, @@ -4796,7 +4797,6 @@ class Eynollah: self.logger.info("Using light text line detection for OCR") self.logger.info("Processing text lines...") - self.device.reset() gc.collect() torch.cuda.empty_cache() From 096def1e9d0b95cf3690734730f675ae5a74c0fd Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 15:13:13 +0200 Subject: [PATCH 351/374] mbreorder/enhancment: fix missing imports (not sure if these models really need that, though) --- src/eynollah/image_enhancer.py | 6 +++--- src/eynollah/mb_ro_on_layout.py | 7 +++---- tests/test_smoke.py | 1 - 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/eynollah/image_enhancer.py b/src/eynollah/image_enhancer.py index 89dde16..9247efe 100644 --- a/src/eynollah/image_enhancer.py +++ b/src/eynollah/image_enhancer.py @@ -6,23 +6,23 @@ from logging import Logger import os import time from typing import Optional -import atexit -from functools import partial from pathlib import Path -from multiprocessing import cpu_count import gc + import cv2 import numpy as np from ocrd_utils import getLogger, tf_disable_interactive_logs import tensorflow as tf from skimage.morphology import skeletonize from tensorflow.keras.models import load_model + from .utils.resize import resize_image from .utils.pil_cv2 import pil2cv from .utils import ( is_image_filename, crop_image_inside_box ) +from .eynollah import PatchEncoder, Patches DPI_THRESHOLD = 298 KERNEL = np.ones((5, 5), np.uint8) diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index 45db8e4..218f973 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -6,25 +6,24 @@ from logging import Logger import os import time from typing import Optional -import atexit -from functools import partial from pathlib import Path -from multiprocessing import cpu_count import xml.etree.ElementTree as ET + import cv2 import numpy as np from ocrd_utils import getLogger import statistics import tensorflow as tf from tensorflow.keras.models import load_model -from .utils.resize import resize_image +from .utils.resize import resize_image from .utils.contour import ( find_new_features_of_contours, return_contours_of_image, return_parent_contours, ) from .utils import is_xml_filename +from .eynollah import PatchEncoder, Patches DPI_THRESHOLD = 298 KERNEL = np.ones((5, 5), np.uint8) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 252213f..e2b323a 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -2,6 +2,5 @@ def test_utils_import(): import eynollah.utils import eynollah.utils.contour import eynollah.utils.drop_capitals - import eynollah.utils.drop_capitals import eynollah.utils.is_nan import eynollah.utils.rotate From 8a2d682e12d8e95414aa53f1e2a9cfea74c778a3 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 16:52:22 +0200 Subject: [PATCH 352/374] fix identifier scope in layout OCR options (w/o full_layout) --- src/eynollah/eynollah.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index aeb01be..7d6229a 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -4726,7 +4726,6 @@ class Eynollah: self.plotter.write_images_into_directory(polygons_of_images, image_page) t_order = time.time() - #if self.full_layout: self.logger.info("Step 4/5: Reading Order Detection") if self.reading_order_machine_based: @@ -4749,46 +4748,41 @@ class Eynollah: boxes_d, textline_mask_tot_d) self.logger.info(f"Detection of reading order took {time.time() - t_order:.1f}s") + ocr_all_textlines = None + ocr_all_textlines_marginals_left = None + ocr_all_textlines_marginals_right = None + ocr_all_textlines_h = None + ocr_all_textlines_drop = None if self.ocr: self.logger.info("Step 4.5/5: OCR Processing") if not self.tr: gc.collect() - if len(all_found_textline_polygons)>0: + if len(all_found_textline_polygons): ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons, all_box_coord, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - else: - ocr_all_textlines = None - if all_found_textline_polygons_marginals_left and len(all_found_textline_polygons_marginals_left)>0: + if len(all_found_textline_polygons_marginals_left): ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons_marginals_left, all_box_coord_marginals_left, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - else: - ocr_all_textlines_marginals_left = None - if all_found_textline_polygons_marginals_right and len(all_found_textline_polygons_marginals_right)>0: + if len(all_found_textline_polygons_marginals_right): ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons_marginals_right, all_box_coord_marginals_right, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - else: - ocr_all_textlines_marginals_right = None - if all_found_textline_polygons_h and len(all_found_textline_polygons)>0: + if self.full_layout and len(all_found_textline_polygons): ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons_h, all_box_coord_h, self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - else: - ocr_all_textlines_h = None - if polygons_of_drop_capitals and len(polygons_of_drop_capitals)>0: + if self.full_layout and len(polygons_of_drop_capitals): ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines( image_page, polygons_of_drop_capitals, np.zeros((len(polygons_of_drop_capitals), 4)), self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) - else: - ocr_all_textlines_drop = None else: if self.light_version: @@ -4805,6 +4799,7 @@ class Eynollah: ind_tot = 0 #cv2.imwrite('./img_out.png', image_page) ocr_all_textlines = [] + # FIXME: what about lines in marginals / headings / drop-capitals here? for indexing, ind_poly_first in enumerate(all_found_textline_polygons): ocr_textline_in_textregion = [] for indexing2, ind_poly in enumerate(ind_poly_first): @@ -4840,12 +4835,6 @@ class Eynollah: ocr_textline_in_textregion.append(text_ocr) ind_tot = ind_tot +1 ocr_all_textlines.append(ocr_textline_in_textregion) - else: - ocr_all_textlines = None - ocr_all_textlines_marginals_left = None - ocr_all_textlines_marginals_right = None - ocr_all_textlines_h = None - ocr_all_textlines_drop = None self.logger.info("Step 5/5: Output Generation") From b3d29bef8961435f85cf0c95ec3dd6c239e74621 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 19:21:07 +0200 Subject: [PATCH 353/374] return_contours_of_interested_region*: rm unused variants --- src/eynollah/eynollah.py | 17 +++++++---------- src/eynollah/utils/contour.py | 33 --------------------------------- 2 files changed, 7 insertions(+), 43 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 7d6229a..e15afd6 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -79,7 +79,6 @@ from .utils.contour import ( get_textregion_contours_in_org_image_light, return_contours_of_image, return_contours_of_interested_region, - return_contours_of_interested_region_by_min_size, return_contours_of_interested_textline, return_parent_contours, dilate_textregion_contours, @@ -4242,14 +4241,11 @@ class Eynollah: all_found_textline_polygons = filter_contours_area_of_image( textline_mask_tot_ea, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001) - M_main_tot = [cv2.moments(all_found_textline_polygons[j]) - for j in range(len(all_found_textline_polygons))] - w_h_textlines = [cv2.boundingRect(all_found_textline_polygons[j])[2:] - for j in range(len(all_found_textline_polygons))] - w_h_textlines = [w_h_textlines[j][0] / float(w_h_textlines[j][1]) for j in range(len(w_h_textlines))] - cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - + cx_main_tot, cy_main_tot = find_center_of_contours(all_found_textline_polygons) + w_h_textlines = [cv2.boundingRect(polygon)[2:] + for polygon in all_found_textline_polygons] + w_h_textlines = [w / float(h) for w, h in w_h_textlines] + all_found_textline_polygons = self.get_textlines_of_a_textregion_sorted( #all_found_textline_polygons[::-1] all_found_textline_polygons, cx_main_tot, cy_main_tot, w_h_textlines) @@ -4677,7 +4673,8 @@ class Eynollah: self.plotter.save_plot_of_layout_all(text_regions_p, image_page) label_img = 4 - polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, label_img) + polygons_of_drop_capitals = return_contours_of_interested_region(text_regions_p, label_img, + min_area=0.00003) ##all_found_textline_polygons = adhere_drop_capital_region_into_corresponding_textline( ##text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, ##all_box_coord, all_box_coord_h, all_found_textline_polygons, all_found_textline_polygons_h, diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 2560846..f998c4d 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -253,39 +253,6 @@ def return_contours_of_image(image): contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) return contours, hierarchy -def return_contours_of_interested_region_by_min_size(region_pre_p, label, min_size=0.00003): - # pixels of images are identified by 5 - if region_pre_p.ndim == 3: - cnts_images = (region_pre_p[:, :, 0] == label) * 1 - else: - cnts_images = (region_pre_p[:, :] == label) * 1 - _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) - - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables( - thresh, contours_imgs, hierarchy, max_area=1, min_area=min_size) - - return contours_imgs - -def return_contours_of_interested_region_by_size(region_pre_p, label, min_area, max_area): - # pixels of images are identified by 5 - if region_pre_p.ndim == 3: - cnts_images = (region_pre_p[:, :, 0] == label) * 1 - else: - cnts_images = (region_pre_p[:, :] == label) * 1 - _, thresh = cv2.threshold(cnts_images.astype(np.uint8), 0, 255, 0) - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables( - thresh, contours_imgs, hierarchy, max_area=max_area, min_area=min_area) - - img_ret = np.zeros((region_pre_p.shape[0], region_pre_p.shape[1])) - img_ret = cv2.fillPoly(img_ret, pts=contours_imgs, color=1) - - return img_ret - def dilate_textline_contours(all_found_textline_polygons): return [[polygon2contour(contour2polygon(contour, dilate=6)) for contour in region] From a144026b2789ae056c7bac619d2e3e2b582e62d6 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 15:13:57 +0200 Subject: [PATCH 354/374] add rough ruff config --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8a63543..2df39b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,3 +51,18 @@ where = ["src"] [tool.coverage.run] branch = true source = ["eynollah"] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +ignore = [ +# disable unused imports +"F401", +# disable import order +"E402", +# disable unused variables +"F841", +# disable bare except +"E722", +] From e1b56d97dab9eed6110fabd85b5ae74b36f18c9f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 8 Oct 2025 17:54:38 +0200 Subject: [PATCH 355/374] CI: lint with ruff --- .github/workflows/test-eynollah.yml | 4 ++++ pyproject.toml | 3 +++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index 759b26c..466e690 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -67,6 +67,10 @@ jobs: make install-dev EXTRAS=OCR,plotting make deps-test EXTRAS=OCR,plotting ls -l models_* + - name: Lint with ruff + uses: astral-sh/ruff-action@v3 + with: + src: "./src" - name: Test with pytest run: make coverage PYTEST_ARGS="-vv --junitxml=pytest.xml" - name: Get coverage results diff --git a/pyproject.toml b/pyproject.toml index 2df39b9..79f9164 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,3 +66,6 @@ ignore = [ # disable bare except "E722", ] + +[tool.ruff.format] +quote-style = "preserve" From cab392601e74e0360e659296f26e1719fb6f742f Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 9 Oct 2025 20:12:06 +0200 Subject: [PATCH 356/374] :memo: update changelog --- CHANGELOG.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6776d6..ab3dd83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,17 @@ Fixed: * `get_smallest_skew`: after shifting search range of rotation angle, use overall best result * Dockerfile: fix CUDA installation (cuDNN contested between Torch and TF due to extra OCR) * OCR: re-instate missing methods and fix `utils_ocr` function calls + * mbreorder/enhancement CLIs: missing imports * :fire: writer: `SeparatorRegion` needs `SeparatorRegionType` (not `ImageRegionType`) f458e3e * tests: switch from `pytest-subtests` to `parametrize` so we can use `pytest-isolate` (so CUDA memory gets freed between tests if running on GPU) +Added: + * test coverage for OCR options in `layout` + * test coverage for table detection in `layout` + * CI linting with ruff + Changed: * polygons: slightly widen for regions and lines, increase for separators @@ -28,7 +34,19 @@ Changed: but use shared memory if necessary, and switch back from `loky` to stdlib, and shutdown in `del()` instead of `atexit` * :fire: OCR: switch CNN-RNN model to `20250930` version compatible with TF 2.12 on CPU, too + * OCR: allow running `-tr` without `-fl`, too * :fire: writer: use `@type='heading'` instead of `'header'` for headings + * :fire: performance gains via refactoring (simplification, less copy-code, vectorization, + avoiding unused calculations, avoiding unnecessary 3-channel image operations) + * :fire: heuristic reading order detection: many improvements + - contour vs splitter box matching: + * contour must be contained in box exactly instead of heuristics + * make fallback center matching, center must be contained in box + - original vs deskewed contour matching: + * same min-area filter on both sides + * similar area score in addition to center proximity + * avoid duplicate and missing mappings by allowing N:M + matches and splitting+joining where necessary * CI: update+improve model caching From c4cb16c2a8e92b0d14b2388ad7a7e8d06e6472fe Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 9 Oct 2025 23:05:50 +0200 Subject: [PATCH 357/374] simplify (`skip_layout_and_reading_order` is already an attr) --- src/eynollah/eynollah.py | 205 +++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 103 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 1b6cee0..3579078 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -2155,7 +2155,7 @@ class Eynollah: page_coord, cont_page) - def get_regions_light_v(self,img,is_image_enhanced, num_col_classifier, skip_layout_and_reading_order=False): + def get_regions_light_v(self,img,is_image_enhanced, num_col_classifier): self.logger.debug("enter get_regions_light_v") t_in = time.time() erosion_hurts = False @@ -2221,110 +2221,110 @@ class Eynollah: #plt.imshwo(self.image_page_org_size) #plt.show() - if not skip_layout_and_reading_order: - #print("inside 2 ", time.time()-t_in) - if num_col_classifier == 1 or num_col_classifier == 2: - if self.image_org.shape[0]/self.image_org.shape[1] > 2.5: - self.logger.debug("resized to %dx%d for %d cols", - img_resized.shape[1], img_resized.shape[0], num_col_classifier) - prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( - True, img_resized, self.model_region_1_2, n_batch_inference=1, - thresholding_for_some_classes_in_light_version=True, - threshold_art_class_layout=self.threshold_art_class_layout) - else: - prediction_regions_org = np.zeros((self.image_org.shape[0], self.image_org.shape[1], 3)) - confidence_matrix = np.zeros((self.image_org.shape[0], self.image_org.shape[1])) - prediction_regions_page, confidence_matrix_page = self.do_prediction_new_concept( - False, self.image_page_org_size, self.model_region_1_2, n_batch_inference=1, - thresholding_for_artificial_class_in_light_version=True, - threshold_art_class_layout=self.threshold_art_class_layout) - ys = slice(*self.page_coord[0:2]) - xs = slice(*self.page_coord[2:4]) - prediction_regions_org[ys, xs] = prediction_regions_page - confidence_matrix[ys, xs] = confidence_matrix_page - - else: - new_h = (900+ (num_col_classifier-3)*100) - img_resized = resize_image(img_bin, int(new_h * img_bin.shape[0] /img_bin.shape[1]), new_h) - self.logger.debug("resized to %dx%d (new_h=%d) for %d cols", - img_resized.shape[1], img_resized.shape[0], new_h, num_col_classifier) - prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( - True, img_resized, self.model_region_1_2, n_batch_inference=2, - thresholding_for_some_classes_in_light_version=True, - threshold_art_class_layout=self.threshold_art_class_layout) - ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, - ###n_batch_inference=3, - ###thresholding_for_some_classes_in_light_version=True) - #print("inside 3 ", time.time()-t_in) - #plt.imshow(prediction_regions_org[:,:,0]) - #plt.show() - - prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - confidence_matrix = resize_image(confidence_matrix, img_height_h, img_width_h ) - img_bin = resize_image(img_bin, img_height_h, img_width_h ) - prediction_regions_org=prediction_regions_org[:,:,0] - - mask_lines_only = (prediction_regions_org[:,:] ==3)*1 - mask_texts_only = (prediction_regions_org[:,:] ==1)*1 - mask_texts_only = mask_texts_only.astype('uint8') - - ##if num_col_classifier == 1 or num_col_classifier == 2: - ###mask_texts_only = cv2.erode(mask_texts_only, KERNEL, iterations=1) - ##mask_texts_only = cv2.dilate(mask_texts_only, KERNEL, iterations=1) - - mask_texts_only = cv2.dilate(mask_texts_only, kernel=np.ones((2,2), np.uint8), iterations=1) - mask_images_only=(prediction_regions_org[:,:] ==2)*1 - - polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) - test_khat = np.zeros(prediction_regions_org.shape) - test_khat = cv2.fillPoly(test_khat, pts=polygons_seplines, color=(1,1,1)) - - #plt.imshow(test_khat[:,:]) - #plt.show() - #for jv in range(1): - #print(jv, hir_seplines[0][232][3]) - #test_khat = np.zeros(prediction_regions_org.shape) - #test_khat = cv2.fillPoly(test_khat, pts = [polygons_seplines[232]], color=(1,1,1)) - #plt.imshow(test_khat[:,:]) - #plt.show() - - polygons_seplines = filter_contours_area_of_image( - mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001, dilate=1) - - test_khat = np.zeros(prediction_regions_org.shape) - test_khat = cv2.fillPoly(test_khat, pts = polygons_seplines, color=(1,1,1)) - - #plt.imshow(test_khat[:,:]) - #plt.show() - #sys.exit() - - polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) - ##polygons_of_only_texts = dilate_textregion_contours(polygons_of_only_texts) - polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) - - text_regions_p_true = np.zeros(prediction_regions_org.shape) - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_lines, color=(3,3,3)) - - text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_texts, color=(1,1,1)) - - textline_mask_tot_ea[(text_regions_p_true==0) | (text_regions_p_true==4) ] = 0 - #plt.imshow(textline_mask_tot_ea) - #plt.show() - #print("inside 4 ", time.time()-t_in) - self.logger.debug("exit get_regions_light_v") - return (text_regions_p_true, - erosion_hurts, - polygons_seplines, - polygons_of_only_texts, - textline_mask_tot_ea, - img_bin, - confidence_matrix) - else: + if self.skip_layout_and_reading_order: img_bin = resize_image(img_bin,img_height_h, img_width_h ) self.logger.debug("exit get_regions_light_v") return None, erosion_hurts, None, None, textline_mask_tot_ea, img_bin, None + #print("inside 2 ", time.time()-t_in) + if num_col_classifier == 1 or num_col_classifier == 2: + if self.image_org.shape[0]/self.image_org.shape[1] > 2.5: + self.logger.debug("resized to %dx%d for %d cols", + img_resized.shape[1], img_resized.shape[0], num_col_classifier) + prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( + True, img_resized, self.model_region_1_2, n_batch_inference=1, + thresholding_for_some_classes_in_light_version=True, + threshold_art_class_layout=self.threshold_art_class_layout) + else: + prediction_regions_org = np.zeros((self.image_org.shape[0], self.image_org.shape[1], 3)) + confidence_matrix = np.zeros((self.image_org.shape[0], self.image_org.shape[1])) + prediction_regions_page, confidence_matrix_page = self.do_prediction_new_concept( + False, self.image_page_org_size, self.model_region_1_2, n_batch_inference=1, + thresholding_for_artificial_class_in_light_version=True, + threshold_art_class_layout=self.threshold_art_class_layout) + ys = slice(*self.page_coord[0:2]) + xs = slice(*self.page_coord[2:4]) + prediction_regions_org[ys, xs] = prediction_regions_page + confidence_matrix[ys, xs] = confidence_matrix_page + + else: + new_h = (900+ (num_col_classifier-3)*100) + img_resized = resize_image(img_bin, int(new_h * img_bin.shape[0] /img_bin.shape[1]), new_h) + self.logger.debug("resized to %dx%d (new_h=%d) for %d cols", + img_resized.shape[1], img_resized.shape[0], new_h, num_col_classifier) + prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( + True, img_resized, self.model_region_1_2, n_batch_inference=2, + thresholding_for_some_classes_in_light_version=True, + threshold_art_class_layout=self.threshold_art_class_layout) + ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, + ###n_batch_inference=3, + ###thresholding_for_some_classes_in_light_version=True) + #print("inside 3 ", time.time()-t_in) + #plt.imshow(prediction_regions_org[:,:,0]) + #plt.show() + + prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) + confidence_matrix = resize_image(confidence_matrix, img_height_h, img_width_h ) + img_bin = resize_image(img_bin, img_height_h, img_width_h ) + prediction_regions_org=prediction_regions_org[:,:,0] + + mask_lines_only = (prediction_regions_org[:,:] ==3)*1 + mask_texts_only = (prediction_regions_org[:,:] ==1)*1 + mask_texts_only = mask_texts_only.astype('uint8') + + ##if num_col_classifier == 1 or num_col_classifier == 2: + ###mask_texts_only = cv2.erode(mask_texts_only, KERNEL, iterations=1) + ##mask_texts_only = cv2.dilate(mask_texts_only, KERNEL, iterations=1) + + mask_texts_only = cv2.dilate(mask_texts_only, kernel=np.ones((2,2), np.uint8), iterations=1) + mask_images_only=(prediction_regions_org[:,:] ==2)*1 + + polygons_seplines, hir_seplines = return_contours_of_image(mask_lines_only) + test_khat = np.zeros(prediction_regions_org.shape) + test_khat = cv2.fillPoly(test_khat, pts=polygons_seplines, color=(1,1,1)) + + #plt.imshow(test_khat[:,:]) + #plt.show() + #for jv in range(1): + #print(jv, hir_seplines[0][232][3]) + #test_khat = np.zeros(prediction_regions_org.shape) + #test_khat = cv2.fillPoly(test_khat, pts = [polygons_seplines[232]], color=(1,1,1)) + #plt.imshow(test_khat[:,:]) + #plt.show() + + polygons_seplines = filter_contours_area_of_image( + mask_lines_only, polygons_seplines, hir_seplines, max_area=1, min_area=0.00001, dilate=1) + + test_khat = np.zeros(prediction_regions_org.shape) + test_khat = cv2.fillPoly(test_khat, pts = polygons_seplines, color=(1,1,1)) + + #plt.imshow(test_khat[:,:]) + #plt.show() + #sys.exit() + + polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) + ##polygons_of_only_texts = dilate_textregion_contours(polygons_of_only_texts) + polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) + + text_regions_p_true = np.zeros(prediction_regions_org.shape) + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_lines, color=(3,3,3)) + + text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_texts, color=(1,1,1)) + + textline_mask_tot_ea[(text_regions_p_true==0) | (text_regions_p_true==4) ] = 0 + #plt.imshow(textline_mask_tot_ea) + #plt.show() + #print("inside 4 ", time.time()-t_in) + self.logger.debug("exit get_regions_light_v") + return (text_regions_p_true, + erosion_hurts, + polygons_seplines, + polygons_of_only_texts, + textline_mask_tot_ea, + img_bin, + confidence_matrix) + def get_regions_from_xy_2models(self,img,is_image_enhanced, num_col_classifier): self.logger.debug("enter get_regions_from_xy_2models") erosion_hurts = False @@ -4226,8 +4226,7 @@ class Eynollah: self.logger.info("Skipping layout analysis and reading order detection") _ ,_, _, _, textline_mask_tot_ea, img_bin_light, _ = \ - self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier, - skip_layout_and_reading_order=self.skip_layout_and_reading_order) + self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier,) page_coord, image_page, textline_mask_tot_ea, img_bin_light, cont_page = \ self.run_graphics_and_columns_without_layout(textline_mask_tot_ea, img_bin_light) From 374818de118dc0292dde789c6c3a233dbce4d83d Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Thu, 9 Oct 2025 23:11:05 +0200 Subject: [PATCH 358/374] :memo: update changelog for 5725e4f --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a0f190..6fd3b2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ Versioned according to [Semantic Versioning](http://semver.org/). Fixed: + * continue processing when no columns detected but text regions exist + * convert marginalia to main text if no main text is present + * reset deskewing angle to 0° when text covers <30% image area and detected angle >45° * :fire: polygons: avoid invalid paths (use `Polygon.buffer()` instead of dilation etc.) * `return_boxes_of_images_by_order_of_reading_new`: avoid Numpy.dtype mismatch, simplify * `return_boxes_of_images_by_order_of_reading_new`: log any exceptions instead of ignoring From 4e9a1618c355a7aeed471c9f63018440adf441cf Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Fri, 10 Oct 2025 03:18:09 +0200 Subject: [PATCH 359/374] layout: refactor model setup, allow loading custom versions - simplify definition of (defaults for) model versions - unify loading of loadable models (depending on mode) - use `self.models` dict instead of `self.model_*` attributes - add `model_versions` kwarg / `--model_version` CLI option --- CHANGELOG.md | 1 + src/eynollah/cli.py | 10 +- src/eynollah/eynollah.py | 362 +++++++++++++++++++-------------------- 3 files changed, 191 insertions(+), 182 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fd3b2e..df1e12e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ f458e3e (so CUDA memory gets freed between tests if running on GPU) Added: + * :fire: `layout` CLI: new option `--model_version` to override default choices * test coverage for OCR options in `layout` * test coverage for table detection in `layout` * CI linting with ruff diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py index 93bb676..c9bad52 100644 --- a/src/eynollah/cli.py +++ b/src/eynollah/cli.py @@ -202,6 +202,13 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low type=click.Path(exists=True, file_okay=False), required=True, ) +@click.option( + "--model_version", + "-mv", + help="override default versions of model categories", + type=(str, str), + multiple=True, +) @click.option( "--save_images", "-si", @@ -373,7 +380,7 @@ def enhancement(image, out, overwrite, dir_in, model, num_col_upper, num_col_low help="Setup a basic console logger", ) -def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, transformer_ocr, batch_size_ocr, num_col_upper, num_col_lower, threshold_art_class_textline, threshold_art_class_layout, skip_layout_and_reading_order, ignore_page_extraction, log_level, setup_logging): +def layout(image, out, overwrite, dir_in, model, model_version, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, transformer_ocr, batch_size_ocr, num_col_upper, num_col_lower, threshold_art_class_textline, threshold_art_class_layout, skip_layout_and_reading_order, ignore_page_extraction, log_level, setup_logging): if setup_logging: console_handler = logging.StreamHandler(sys.stdout) console_handler.setLevel(logging.INFO) @@ -404,6 +411,7 @@ def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_ assert bool(image) != bool(dir_in), "Either -i (single input) or -di (directory) must be provided, but not both." eynollah = Eynollah( model, + model_versions=model_version, extract_only_images=extract_only_images, enable_plotting=enable_plotting, allow_enhancement=allow_enhancement, diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 3579078..0992c8c 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -19,7 +19,7 @@ import math import os import sys import time -from typing import Optional +from typing import Dict, List, Optional, Tuple import atexit import warnings from functools import partial @@ -180,7 +180,6 @@ class Patches(layers.Layer): }) return config - class PatchEncoder(layers.Layer): def __init__(self, **kwargs): super(PatchEncoder, self).__init__() @@ -208,6 +207,7 @@ class Eynollah: def __init__( self, dir_models : str, + model_versions: List[Tuple[str, str]] = [], extract_only_images : bool =False, enable_plotting : bool = False, allow_enhancement : bool = False, @@ -254,6 +254,10 @@ class Eynollah: self.skip_layout_and_reading_order = skip_layout_and_reading_order self.ocr = do_ocr self.tr = transformer_ocr + if not batch_size_ocr: + self.b_s_ocr = 8 + else: + self.b_s_ocr = int(batch_size_ocr) if num_col_upper: self.num_col_upper = int(num_col_upper) else: @@ -275,69 +279,6 @@ class Eynollah: self.threshold_art_class_textline = float(threshold_art_class_textline) else: self.threshold_art_class_textline = 0.1 - - self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" - self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" - self.model_dir_of_col_classifier = dir_models + "/eynollah-column-classifier_20210425" - self.model_region_dir_p = dir_models + "/eynollah-main-regions-aug-scaling_20210425" - self.model_region_dir_p2 = dir_models + "/eynollah-main-regions-aug-rotation_20210425" - #"/modelens_full_lay_1_3_031124" - #"/modelens_full_lay_13__3_19_241024" - #"/model_full_lay_13_241024" - #"/modelens_full_lay_13_17_231024" - #"/modelens_full_lay_1_2_221024" - #"/eynollah-full-regions-1column_20210425" - self.model_region_dir_fully_np = dir_models + "/modelens_full_lay_1__4_3_091124" - #self.model_region_dir_fully = dir_models + "/eynollah-full-regions-3+column_20210425" - self.model_page_dir = dir_models + "/model_eynollah_page_extraction_20250915" - self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" - self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" - self.model_region_dir_p_ens_light_only_images_extraction = (dir_models + - "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - ) - self.model_reading_order_dir = (dir_models + - "/model_eynollah_reading_order_20250824" - #"/model_mb_ro_aug_ens_11" - #"/model_step_3200000_mb_ro" - #"/model_ens_reading_order_machine_based" - #"/model_mb_ro_aug_ens_8" - #"/model_ens_reading_order_machine_based" - ) - #"/modelens_12sp_elay_0_3_4__3_6_n" - #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" - #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" - #"/modelens_1_2_4_5_early_lay_1_2_spaltige" - #"/model_3_eraly_layout_no_patches_1_2_spaltige" - self.model_region_dir_p_1_2_sp_np = dir_models + "/modelens_e_l_all_sp_0_1_2_3_4_171024" - ##self.model_region_dir_fully_new = dir_models + "/model_2_full_layout_new_trans" - #"/modelens_full_lay_1_3_031124" - #"/modelens_full_lay_13__3_19_241024" - #"/model_full_lay_13_241024" - #"/modelens_full_lay_13_17_231024" - #"/modelens_full_lay_1_2_221024" - #"/modelens_full_layout_24_till_28" - #"/model_2_full_layout_new_trans" - self.model_region_dir_fully = dir_models + "/modelens_full_lay_1__4_3_091124" - if self.textline_light: - #"/modelens_textline_1_4_16092024" - #"/model_textline_ens_3_4_5_6_artificial" - #"/modelens_textline_1_3_4_20240915" - #"/model_textline_ens_3_4_5_6_artificial" - #"/modelens_textline_9_12_13_14_15" - #"/eynollah-textline_light_20210425" - self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024" - else: - #"/eynollah-textline_20210425" - self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024" - if self.ocr and self.tr: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_trocr_20250919" - elif self.ocr and not self.tr: - self.model_ocr_dir = dir_models + "/model_eynollah_ocr_cnnrnn_20250930" - if self.tables: - if self.light_version: - self.model_table_dir = dir_models + "/modelens_table_0t4_201124" - else: - self.model_table_dir = dir_models + "/eynollah-tables_20210319" t_start = time.time() @@ -356,28 +297,124 @@ class Eynollah: self.logger.warning("no GPU device available") self.logger.info("Loading models...") - - self.model_page = self.our_load_model(self.model_page_dir) - self.model_classifier = self.our_load_model(self.model_dir_of_col_classifier) - self.model_bin = self.our_load_model(self.model_dir_of_binarization) - if self.extract_only_images: - self.model_region = self.our_load_model(self.model_region_dir_p_ens_light_only_images_extraction) - else: - self.model_textline = self.our_load_model(self.model_textline_dir) + self.setup_models(dir_models, model_versions) + self.logger.info(f"Model initialization complete ({time.time() - t_start:.1f}s)") + + @staticmethod + def our_load_model(model_file, basedir=""): + if basedir: + model_file = os.path.join(basedir, model_file) + if model_file.endswith('.h5') and Path(model_file[:-3]).exists(): + # prefer SavedModel over HDF5 format if it exists + model_file = model_file[:-3] + try: + model = load_model(model_file, compile=False) + except: + model = load_model(model_file, compile=False, custom_objects={ + "PatchEncoder": PatchEncoder, "Patches": Patches}) + return model + + def setup_models(self, basedir: Path, model_versions: List[Tuple[str, str]] = []): + self.model_versions = { + "enhancement": "eynollah-enhancement_20210425", + "binarization": "eynollah-binarization_20210425", + "col_classifier": "eynollah-column-classifier_20210425", + "page": "model_eynollah_page_extraction_20250915", + #?: "eynollah-main-regions-aug-scaling_20210425", + "region": ( # early layout + "eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" if self.extract_only_images else + "eynollah-main-regions_20220314" if self.light_version else + "eynollah-main-regions-ensembled_20210425"), + "region_p2": ( # early layout, non-light, 2nd part + "eynollah-main-regions-aug-rotation_20210425"), + "region_1_2": ( # early layout, light, 1-or-2-column + #"modelens_12sp_elay_0_3_4__3_6_n" + #"modelens_earlylayout_12spaltige_2_3_5_6_7_8" + #"modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" + #"modelens_1_2_4_5_early_lay_1_2_spaltige" + #"model_3_eraly_layout_no_patches_1_2_spaltige" + "modelens_e_l_all_sp_0_1_2_3_4_171024"), + "region_fl_np": ( # full layout / no patches + #"modelens_full_lay_1_3_031124" + #"modelens_full_lay_13__3_19_241024" + #"model_full_lay_13_241024" + #"modelens_full_lay_13_17_231024" + #"modelens_full_lay_1_2_221024" + #"eynollah-full-regions-1column_20210425" + "modelens_full_lay_1__4_3_091124"), + "region_fl": ( # full layout / with patches + #"eynollah-full-regions-3+column_20210425" + ##"model_2_full_layout_new_trans" + #"modelens_full_lay_1_3_031124" + #"modelens_full_lay_13__3_19_241024" + #"model_full_lay_13_241024" + #"modelens_full_lay_13_17_231024" + #"modelens_full_lay_1_2_221024" + #"modelens_full_layout_24_till_28" + #"model_2_full_layout_new_trans" + "modelens_full_lay_1__4_3_091124"), + "reading_order": ( + #"model_mb_ro_aug_ens_11" + #"model_step_3200000_mb_ro" + #"model_ens_reading_order_machine_based" + #"model_mb_ro_aug_ens_8" + #"model_ens_reading_order_machine_based" + "model_eynollah_reading_order_20250824"), + "textline": ( + #"modelens_textline_1_4_16092024" + #"model_textline_ens_3_4_5_6_artificial" + #"modelens_textline_1_3_4_20240915" + #"model_textline_ens_3_4_5_6_artificial" + #"modelens_textline_9_12_13_14_15" + #"eynollah-textline_light_20210425" + "modelens_textline_0_1__2_4_16092024" if self.textline_light else + #"eynollah-textline_20210425" + "modelens_textline_0_1__2_4_16092024"), + "table": ( + None if not self.tables else + "modelens_table_0t4_201124" if self.light_version else + "eynollah-tables_20210319"), + "ocr": ( + None if not self.ocr else + "model_eynollah_ocr_trocr_20250919" if self.tr else + "model_eynollah_ocr_cnnrnn_20250930") + } + # override defaults from CLI + for key, val in model_versions: + assert key in self.model_versions, "unknown model category '%s'" % key + self.logger.warning("overriding default model %s version %s to %s", key, self.model_versions[key], val) + self.model_versions[key] = val + # load models, depending on modes + loadable = [ + "col_classifier", + "binarization", + "page", + "region" + ] + if not self.extract_only_images: + loadable.append("textline") if self.light_version: - self.model_region = self.our_load_model(self.model_region_dir_p_ens_light) - self.model_region_1_2 = self.our_load_model(self.model_region_dir_p_1_2_sp_np) + loadable.append("region_1_2") else: - self.model_region = self.our_load_model(self.model_region_dir_p_ens) - self.model_region_p2 = self.our_load_model(self.model_region_dir_p2) - self.model_enhancement = self.our_load_model(self.model_dir_of_enhancement) - ###self.model_region_fl_new = self.our_load_model(self.model_region_dir_fully_new) - self.model_region_fl_np = self.our_load_model(self.model_region_dir_fully_np) - self.model_region_fl = self.our_load_model(self.model_region_dir_fully) + loadable.append("region_p2") + # if self.allow_enhancement:? + loadable.append("enhancement") + if self.full_layout: + loadable.extend(["region_fl_np", + "region_fl"]) if self.reading_order_machine_based: - self.model_reading_order = self.our_load_model(self.model_reading_order_dir) - if self.ocr and self.tr: - self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) + loadable.append("reading_order") + if self.tables: + loadable.append("table") + + self.models = {name: self.our_load_model(self.model_versions[name], basedir) + for name in loadable + } + + if self.ocr: + ocr_model_dir = os.path.join(basedir, self.model_versions["ocr"]) + if self.tr: + self.models["ocr"] = VisionEncoderDecoderModel.from_pretrained(ocr_model_dir) if torch.cuda.is_available(): self.logger.info("Using GPU acceleration") self.device = torch.device("cuda:0") @@ -386,54 +423,29 @@ class Eynollah: self.device = torch.device("cpu") #self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") - elif self.ocr and not self.tr: - model_ocr = load_model(self.model_ocr_dir , compile=False) - - self.prediction_model = tf.keras.models.Model( - model_ocr.get_layer(name = "image").input, - model_ocr.get_layer(name = "dense2").output) - if not batch_size_ocr: - self.b_s_ocr = 8 - else: - self.b_s_ocr = int(batch_size_ocr) + else: + ocr_model = load_model(ocr_model_dir, compile=False) + self.models["ocr"] = tf.keras.models.Model( + ocr_model.get_layer(name = "image").input, + ocr_model.get_layer(name = "dense2").output) - with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: + with open(os.path.join(ocr_model_dir, "characters_org.txt"), "r") as config_file: characters = json.load(config_file) - - AUTOTUNE = tf.data.AUTOTUNE - # Mapping characters to integers. char_to_num = StringLookup(vocabulary=list(characters), mask_token=None) - # Mapping integers back to original characters. self.num_to_char = StringLookup( vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True ) - - if self.tables: - self.model_table = self.our_load_model(self.model_table_dir) - - self.logger.info(f"Model initialization complete ({time.time() - t_start:.1f}s)") def __del__(self): if hasattr(self, 'executor') and getattr(self, 'executor'): self.executor.shutdown() - for model_name in ['model_page', - 'model_classifier', - 'model_bin', - 'model_enhancement', - 'model_region', - 'model_region_1_2', - 'model_region_p2', - 'model_region_fl_np', - 'model_region_fl', - 'model_textline', - 'model_reading_order', - 'model_table', - 'model_ocr', - 'processor']: - if hasattr(self, model_name) and getattr(self, model_name): - delattr(self, model_name) + self.executor = None + if hasattr(self, 'models') and getattr(self, 'models'): + for model_name in list(self.models): + if self.models[model_name]: + del self.models[model_name] def cache_images(self, image_filename=None, image_pil=None, dpi=None): ret = {} @@ -480,8 +492,8 @@ class Eynollah: def predict_enhancement(self, img): self.logger.debug("enter predict_enhancement") - img_height_model = self.model_enhancement.layers[-1].output_shape[1] - img_width_model = self.model_enhancement.layers[-1].output_shape[2] + img_height_model = self.models["enhancement"].layers[-1].output_shape[1] + img_width_model = self.models["enhancement"].layers[-1].output_shape[2] if img.shape[0] < img_height_model: img = cv2.resize(img, (img.shape[1], img_width_model), interpolation=cv2.INTER_NEAREST) if img.shape[1] < img_width_model: @@ -522,7 +534,7 @@ class Eynollah: index_y_d = img_h - img_height_model img_patch = img[np.newaxis, index_y_d:index_y_u, index_x_d:index_x_u, :] - label_p_pred = self.model_enhancement.predict(img_patch, verbose=0) + label_p_pred = self.models["enhancement"].predict(img_patch, verbose=0) seg = label_p_pred[0, :, :, :] * 255 if i == 0 and j == 0: @@ -697,7 +709,7 @@ class Eynollah: img_in[0, :, :, 1] = img_1ch[:, :] img_in[0, :, :, 2] = img_1ch[:, :] - label_p_pred = self.model_classifier.predict(img_in, verbose=0) + label_p_pred = self.models["col_classifier"].predict(img_in, verbose=0) num_col = np.argmax(label_p_pred[0]) + 1 self.logger.info("Found %s columns (%s)", num_col, label_p_pred) @@ -715,7 +727,7 @@ class Eynollah: self.logger.info("Detected %s DPI", dpi) if self.input_binary: img = self.imread() - prediction_bin = self.do_prediction(True, img, self.model_bin, n_batch_inference=5) + prediction_bin = self.do_prediction(True, img, self.models["binarization"], n_batch_inference=5) prediction_bin = 255 * (prediction_bin[:,:,0] == 0) prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2).astype(np.uint8) img= np.copy(prediction_bin) @@ -755,7 +767,7 @@ class Eynollah: img_in[0, :, :, 1] = img_1ch[:, :] img_in[0, :, :, 2] = img_1ch[:, :] - label_p_pred = self.model_classifier.predict(img_in, verbose=0) + label_p_pred = self.models["col_classifier"].predict(img_in, verbose=0) num_col = np.argmax(label_p_pred[0]) + 1 elif (self.num_col_upper and self.num_col_lower) and (self.num_col_upper!=self.num_col_lower): @@ -776,7 +788,7 @@ class Eynollah: img_in[0, :, :, 1] = img_1ch[:, :] img_in[0, :, :, 2] = img_1ch[:, :] - label_p_pred = self.model_classifier.predict(img_in, verbose=0) + label_p_pred = self.models["col_classifier"].predict(img_in, verbose=0) num_col = np.argmax(label_p_pred[0]) + 1 if num_col > self.num_col_upper: @@ -1628,7 +1640,7 @@ class Eynollah: cont_page = [] if not self.ignore_page_extraction: img = np.copy(self.image)#cv2.GaussianBlur(self.image, (5, 5), 0) - img_page_prediction = self.do_prediction(False, img, self.model_page) + img_page_prediction = self.do_prediction(False, img, self.models["page"]) imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) _, thresh = cv2.threshold(imgray, 0, 255, 0) ##thresh = cv2.dilate(thresh, KERNEL, iterations=3) @@ -1676,7 +1688,7 @@ class Eynollah: else: img = self.imread() img = cv2.GaussianBlur(img, (5, 5), 0) - img_page_prediction = self.do_prediction(False, img, self.model_page) + img_page_prediction = self.do_prediction(False, img, self.models["page"]) imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) _, thresh = cv2.threshold(imgray, 0, 255, 0) @@ -1702,7 +1714,7 @@ class Eynollah: self.logger.debug("enter extract_text_regions") img_height_h = img.shape[0] img_width_h = img.shape[1] - model_region = self.model_region_fl if patches else self.model_region_fl_np + model_region = self.models["region_fl"] if patches else self.models["region_fl_np"] if self.light_version: thresholding_for_fl_light_version = True @@ -1737,7 +1749,7 @@ class Eynollah: self.logger.debug("enter extract_text_regions") img_height_h = img.shape[0] img_width_h = img.shape[1] - model_region = self.model_region_fl if patches else self.model_region_fl_np + model_region = self.models["region_fl"] if patches else self.models["region_fl_np"] if not patches: img = otsu_copy_binary(img) @@ -1958,14 +1970,14 @@ class Eynollah: img_w = img_org.shape[1] img = resize_image(img_org, int(img_org.shape[0] * scaler_h), int(img_org.shape[1] * scaler_w)) - prediction_textline = self.do_prediction(use_patches, img, self.model_textline, + prediction_textline = self.do_prediction(use_patches, img, self.models["textline"], marginal_of_patch_percent=0.15, n_batch_inference=3, thresholding_for_artificial_class_in_light_version=self.textline_light, threshold_art_class_textline=self.threshold_art_class_textline) #if not self.textline_light: #if num_col_classifier==1: - #prediction_textline_nopatch = self.do_prediction(False, img, self.model_textline) + #prediction_textline_nopatch = self.do_prediction(False, img, self.models["textline"]) #prediction_textline[:,:][prediction_textline_nopatch[:,:]==0] = 0 prediction_textline = resize_image(prediction_textline, img_h, img_w) @@ -2036,7 +2048,7 @@ class Eynollah: #cv2.imwrite('prediction_textline2.png', prediction_textline[:,:,0]) - prediction_textline_longshot = self.do_prediction(False, img, self.model_textline) + prediction_textline_longshot = self.do_prediction(False, img, self.models["textline"]) prediction_textline_longshot_true_size = resize_image(prediction_textline_longshot, img_h, img_w) @@ -2069,7 +2081,7 @@ class Eynollah: img_h_new = int(img.shape[0] / float(img.shape[1]) * img_w_new) img_resized = resize_image(img,img_h_new, img_w_new ) - prediction_regions_org, _ = self.do_prediction_new_concept(True, img_resized, self.model_region) + prediction_regions_org, _ = self.do_prediction_new_concept(True, img_resized, self.models["region"]) prediction_regions_org = resize_image(prediction_regions_org,img_height_h, img_width_h ) image_page, page_coord, cont_page = self.extract_page() @@ -2185,7 +2197,7 @@ class Eynollah: #if self.input_binary: #img_bin = np.copy(img_resized) ###if (not self.input_binary and self.full_layout) or (not self.input_binary and num_col_classifier >= 30): - ###prediction_bin = self.do_prediction(True, img_resized, self.model_bin, n_batch_inference=5) + ###prediction_bin = self.do_prediction(True, img_resized, self.models["binarization"], n_batch_inference=5) ####print("inside bin ", time.time()-t_bin) ###prediction_bin=prediction_bin[:,:,0] @@ -2200,7 +2212,7 @@ class Eynollah: ###else: ###img_bin = np.copy(img_resized) if (self.ocr and self.tr) and not self.input_binary: - prediction_bin = self.do_prediction(True, img_resized, self.model_bin, n_batch_inference=5) + prediction_bin = self.do_prediction(True, img_resized, self.models["binarization"], n_batch_inference=5) prediction_bin = 255 * (prediction_bin[:,:,0] == 0) prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) prediction_bin = prediction_bin.astype(np.uint16) @@ -2232,14 +2244,14 @@ class Eynollah: self.logger.debug("resized to %dx%d for %d cols", img_resized.shape[1], img_resized.shape[0], num_col_classifier) prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( - True, img_resized, self.model_region_1_2, n_batch_inference=1, + True, img_resized, self.models["region_1_2"], n_batch_inference=1, thresholding_for_some_classes_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) else: prediction_regions_org = np.zeros((self.image_org.shape[0], self.image_org.shape[1], 3)) confidence_matrix = np.zeros((self.image_org.shape[0], self.image_org.shape[1])) prediction_regions_page, confidence_matrix_page = self.do_prediction_new_concept( - False, self.image_page_org_size, self.model_region_1_2, n_batch_inference=1, + False, self.image_page_org_size, self.models["region_1_2"], n_batch_inference=1, thresholding_for_artificial_class_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) ys = slice(*self.page_coord[0:2]) @@ -2253,10 +2265,10 @@ class Eynollah: self.logger.debug("resized to %dx%d (new_h=%d) for %d cols", img_resized.shape[1], img_resized.shape[0], new_h, num_col_classifier) prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( - True, img_resized, self.model_region_1_2, n_batch_inference=2, + True, img_resized, self.models["region_1_2"], n_batch_inference=2, thresholding_for_some_classes_in_light_version=True, threshold_art_class_layout=self.threshold_art_class_layout) - ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, + ###prediction_regions_org = self.do_prediction(True, img_bin, self.models["region"], ###n_batch_inference=3, ###thresholding_for_some_classes_in_light_version=True) #print("inside 3 ", time.time()-t_in) @@ -2336,7 +2348,7 @@ class Eynollah: ratio_x=1 img = resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - prediction_regions_org_y = self.do_prediction(True, img, self.model_region) + prediction_regions_org_y = self.do_prediction(True, img, self.models["region"]) prediction_regions_org_y = resize_image(prediction_regions_org_y, img_height_h, img_width_h ) #plt.imshow(prediction_regions_org_y[:,:,0]) @@ -2351,7 +2363,7 @@ class Eynollah: _, _ = find_num_col(img_only_regions, num_col_classifier, self.tables, multiplier=6.0) img = resize_image(img_org, int(img_org.shape[0]), int(img_org.shape[1]*(1.2 if is_image_enhanced else 1))) - prediction_regions_org = self.do_prediction(True, img, self.model_region) + prediction_regions_org = self.do_prediction(True, img, self.models["region"]) prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) prediction_regions_org=prediction_regions_org[:,:,0] @@ -2359,7 +2371,7 @@ class Eynollah: img = resize_image(img_org, int(img_org.shape[0]), int(img_org.shape[1])) - prediction_regions_org2 = self.do_prediction(True, img, self.model_region_p2, marginal_of_patch_percent=0.2) + prediction_regions_org2 = self.do_prediction(True, img, self.models["region_p2"], marginal_of_patch_percent=0.2) prediction_regions_org2=resize_image(prediction_regions_org2, img_height_h, img_width_h ) mask_zeros2 = (prediction_regions_org2[:,:,0] == 0) @@ -2383,7 +2395,7 @@ class Eynollah: if self.input_binary: prediction_bin = np.copy(img_org) else: - prediction_bin = self.do_prediction(True, img_org, self.model_bin, n_batch_inference=5) + prediction_bin = self.do_prediction(True, img_org, self.models["binarization"], n_batch_inference=5) prediction_bin = resize_image(prediction_bin, img_height_h, img_width_h ) prediction_bin = 255 * (prediction_bin[:,:,0]==0) prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) @@ -2393,7 +2405,7 @@ class Eynollah: img = resize_image(prediction_bin, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - prediction_regions_org = self.do_prediction(True, img, self.model_region) + prediction_regions_org = self.do_prediction(True, img, self.models["region"]) prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) prediction_regions_org=prediction_regions_org[:,:,0] @@ -2420,7 +2432,7 @@ class Eynollah: except: if self.input_binary: prediction_bin = np.copy(img_org) - prediction_bin = self.do_prediction(True, img_org, self.model_bin, n_batch_inference=5) + prediction_bin = self.do_prediction(True, img_org, self.models["binarization"], n_batch_inference=5) prediction_bin = resize_image(prediction_bin, img_height_h, img_width_h ) prediction_bin = 255 * (prediction_bin[:,:,0]==0) prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) @@ -2431,14 +2443,14 @@ class Eynollah: img = resize_image(prediction_bin, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - prediction_regions_org = self.do_prediction(True, img, self.model_region) + prediction_regions_org = self.do_prediction(True, img, self.models["region"]) prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) prediction_regions_org=prediction_regions_org[:,:,0] #mask_lines_only=(prediction_regions_org[:,:]==3)*1 #img = resize_image(img_org, int(img_org.shape[0]*1), int(img_org.shape[1]*1)) - #prediction_regions_org = self.do_prediction(True, img, self.model_region) + #prediction_regions_org = self.do_prediction(True, img, self.models["region"]) #prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) #prediction_regions_org = prediction_regions_org[:,:,0] #prediction_regions_org[(prediction_regions_org[:,:] == 1) & (mask_zeros_y[:,:] == 1)]=0 @@ -2809,13 +2821,13 @@ class Eynollah: img_width_h = img_org.shape[1] patches = False if self.light_version: - prediction_table, _ = self.do_prediction_new_concept(patches, img, self.model_table) + prediction_table, _ = self.do_prediction_new_concept(patches, img, self.models["table"]) prediction_table = prediction_table.astype(np.int16) return prediction_table[:,:,0] else: if num_col_classifier < 4 and num_col_classifier > 2: - prediction_table = self.do_prediction(patches, img, self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img[:,:,:], -1), self.model_table) + prediction_table = self.do_prediction(patches, img, self.models["table"]) + pre_updown = self.do_prediction(patches, cv2.flip(img[:,:,:], -1), self.models["table"]) pre_updown = cv2.flip(pre_updown, -1) prediction_table[:,:,0][pre_updown[:,:,0]==1]=1 @@ -2834,8 +2846,8 @@ class Eynollah: xs = slice(w_start, w_start + img.shape[1]) img_new[ys, xs] = img - prediction_ext = self.do_prediction(patches, img_new, self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img_new[:,:,:], -1), self.model_table) + prediction_ext = self.do_prediction(patches, img_new, self.models["table"]) + pre_updown = self.do_prediction(patches, cv2.flip(img_new[:,:,:], -1), self.models["table"]) pre_updown = cv2.flip(pre_updown, -1) prediction_table = prediction_ext[ys, xs] @@ -2856,8 +2868,8 @@ class Eynollah: xs = slice(w_start, w_start + img.shape[1]) img_new[ys, xs] = img - prediction_ext = self.do_prediction(patches, img_new, self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img_new[:,:,:], -1), self.model_table) + prediction_ext = self.do_prediction(patches, img_new, self.models["table"]) + pre_updown = self.do_prediction(patches, cv2.flip(img_new[:,:,:], -1), self.models["table"]) pre_updown = cv2.flip(pre_updown, -1) prediction_table = prediction_ext[ys, xs] @@ -2869,10 +2881,10 @@ class Eynollah: prediction_table = np.zeros(img.shape) img_w_half = img.shape[1] // 2 - pre1 = self.do_prediction(patches, img[:,0:img_w_half,:], self.model_table) - pre2 = self.do_prediction(patches, img[:,img_w_half:,:], self.model_table) - pre_full = self.do_prediction(patches, img[:,:,:], self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img[:,:,:], -1), self.model_table) + pre1 = self.do_prediction(patches, img[:,0:img_w_half,:], self.models["table"]) + pre2 = self.do_prediction(patches, img[:,img_w_half:,:], self.models["table"]) + pre_full = self.do_prediction(patches, img[:,:,:], self.models["table"]) + pre_updown = self.do_prediction(patches, cv2.flip(img[:,:,:], -1), self.models["table"]) pre_updown = cv2.flip(pre_updown, -1) prediction_table_full_erode = cv2.erode(pre_full[:,:,0], KERNEL, iterations=4) @@ -3474,18 +3486,6 @@ class Eynollah: regions_without_separators_d, regions_fully, regions_without_separators, polygons_of_marginals, contours_tables) - @staticmethod - def our_load_model(model_file): - if model_file.endswith('.h5') and Path(model_file[:-3]).exists(): - # prefer SavedModel over HDF5 format if it exists - model_file = model_file[:-3] - try: - model = load_model(model_file, compile=False) - except: - model = load_model(model_file, compile=False, custom_objects={ - "PatchEncoder": PatchEncoder, "Patches": Patches}) - return model - def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p): height1 =672#448 @@ -3676,7 +3676,7 @@ class Eynollah: tot_counter += 1 batch.append(j) if tot_counter % inference_bs == 0 or tot_counter == len(ij_list): - y_pr = self.model_reading_order.predict(input_1 , verbose=0) + y_pr = self.models["reading_order"].predict(input_1 , verbose=0) for jb, j in enumerate(batch): if y_pr[jb][0]>=0.5: post_list.append(j) @@ -4259,7 +4259,7 @@ class Eynollah: gc.collect() ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons, np.zeros((len(all_found_textline_polygons), 4)), - self.prediction_model, self.b_s_ocr, self.num_to_char, textline_light=True) + self.models["ocr"], self.b_s_ocr, self.num_to_char, textline_light=True) else: ocr_all_textlines = None @@ -4768,27 +4768,27 @@ class Eynollah: if len(all_found_textline_polygons): ocr_all_textlines = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons, all_box_coord, - self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + self.models["ocr"], self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) if len(all_found_textline_polygons_marginals_left): ocr_all_textlines_marginals_left = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons_marginals_left, all_box_coord_marginals_left, - self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + self.models["ocr"], self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) if len(all_found_textline_polygons_marginals_right): ocr_all_textlines_marginals_right = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons_marginals_right, all_box_coord_marginals_right, - self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + self.models["ocr"], self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) if self.full_layout and len(all_found_textline_polygons): ocr_all_textlines_h = return_rnn_cnn_ocr_of_given_textlines( image_page, all_found_textline_polygons_h, all_box_coord_h, - self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + self.models["ocr"], self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) if self.full_layout and len(polygons_of_drop_capitals): ocr_all_textlines_drop = return_rnn_cnn_ocr_of_given_textlines( image_page, polygons_of_drop_capitals, np.zeros((len(polygons_of_drop_capitals), 4)), - self.prediction_model, self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) + self.models["ocr"], self.b_s_ocr, self.num_to_char, self.textline_light, self.curved_line) else: if self.light_version: @@ -4800,7 +4800,7 @@ class Eynollah: gc.collect() torch.cuda.empty_cache() - self.model_ocr.to(self.device) + self.models["ocr"].to(self.device) ind_tot = 0 #cv2.imwrite('./img_out.png', image_page) @@ -4837,7 +4837,7 @@ class Eynollah: img_croped = img_poly_on_img[y:y+h, x:x+w, :] #cv2.imwrite('./extracted_lines/'+str(ind_tot)+'.jpg', img_croped) text_ocr = self.return_ocr_of_textline_without_common_section( - img_croped, self.model_ocr, self.processor, self.device, w, h2w_ratio, ind_tot) + img_croped, self.models["ocr"], self.processor, self.device, w, h2w_ratio, ind_tot) ocr_textline_in_textregion.append(text_ocr) ind_tot = ind_tot +1 ocr_all_textlines.append(ocr_textline_in_textregion) From 2056a8bdb9aff8895235f36f2ddf11a42b0469a3 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 10 Oct 2025 16:32:47 +0200 Subject: [PATCH 360/374] :package: v0.6.0rc1 --- CHANGELOG.md | 3 +++ src/eynollah/ocrd-tool.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df1e12e..d0ad43c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +## [0.6.0rc1] - 2025-10-10 + Fixed: * continue processing when no columns detected but text regions exist @@ -289,6 +291,7 @@ Fixed: Initial release +[0.6.0rc1]: ../../compare/v0.6.0rc1...v0.5.0 [0.5.0]: ../../compare/v0.5.0...v0.4.0 [0.4.0]: ../../compare/v0.4.0...v0.3.1 [0.3.1]: ../../compare/v0.3.1...v0.3.0 diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index 5d89c92..2ae4ead 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -1,5 +1,5 @@ { - "version": "0.5.0", + "version": "0.6.0rc1", "git_url": "https://github.com/qurator-spk/eynollah", "dockerhub": "ocrd/eynollah", "tools": { From 745cf3be48ad6d5fee9c6297e50ea2d52d7f8fd2 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 10 Oct 2025 16:39:16 +0200 Subject: [PATCH 361/374] XML encoding should be utf-8 not utf8 ... and should use OCR-D's generateDS PAGE API consistently --- src/eynollah/eynollah.py | 4 ++-- src/eynollah/mb_ro_on_layout.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0992c8c..94bd10c 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -5284,7 +5284,7 @@ class Eynollah_ocr: ##unicode_textpage.text = tot_page_text ET.register_namespace("",name_space) - tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) + tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf-8",default_namespace=None) else: ###max_len = 280#512#280#512 ###padding_token = 1500#299#1500#299 @@ -5833,5 +5833,5 @@ class Eynollah_ocr: ##unicode_textpage.text = tot_page_text ET.register_namespace("",name_space) - tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) + tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf-8",default_namespace=None) #print("Job done in %.1fs", time.time() - t0) diff --git a/src/eynollah/mb_ro_on_layout.py b/src/eynollah/mb_ro_on_layout.py index 218f973..1b991ae 100644 --- a/src/eynollah/mb_ro_on_layout.py +++ b/src/eynollah/mb_ro_on_layout.py @@ -805,7 +805,7 @@ class machine_based_reading_order_on_layout: tree_xml.write(os.path.join(dir_out, file_name+'.xml'), xml_declaration=True, method='xml', - encoding="utf8", + encoding="utf-8", default_namespace=None) #sys.exit() From e8b7212f36af40c536bdf3607d53d6c60460b129 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 14 Oct 2025 14:16:39 +0200 Subject: [PATCH 362/374] `polygon2contour`: avoid uint for coords (introduced in a433c736 to make consistent with `filter_contours_area_of_image`, but actually np.uint is prone to create overflows downstream) --- src/eynollah/utils/contour.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index f998c4d..21068b3 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -276,7 +276,7 @@ def contour2polygon(contour: Union[np.ndarray, Sequence[Sequence[Sequence[Number def polygon2contour(polygon: Polygon) -> np.ndarray: polygon = np.array(polygon.exterior.coords[:-1], dtype=int) - return np.maximum(0, polygon).astype(np.uint)[:, np.newaxis] + return np.maximum(0, polygon).astype(int)[:, np.newaxis] def make_intersection(poly1, poly2): interp = poly1.intersection(poly2) From 8299e7009a569c0c3c82e603df245c730f4f52b4 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Tue, 14 Oct 2025 14:23:29 +0200 Subject: [PATCH 363/374] `setup_models`: avoid unnecessarily loading `region_fl` --- src/eynollah/eynollah.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py index 0992c8c..6367c91 100644 --- a/src/eynollah/eynollah.py +++ b/src/eynollah/eynollah.py @@ -385,6 +385,8 @@ class Eynollah: self.logger.warning("overriding default model %s version %s to %s", key, self.model_versions[key], val) self.model_versions[key] = val # load models, depending on modes + # (note: loading too many models can cause OOM on GPU/CUDA, + # thus, we try set up the minimal configuration for the current mode) loadable = [ "col_classifier", "binarization", @@ -400,8 +402,8 @@ class Eynollah: # if self.allow_enhancement:? loadable.append("enhancement") if self.full_layout: - loadable.extend(["region_fl_np", - "region_fl"]) + loadable.append("region_fl_np") + #loadable.append("region_fl") if self.reading_order_machine_based: loadable.append("reading_order") if self.tables: From 2febf534797eaa5be35caf16d7965c3ac39bdd39 Mon Sep 17 00:00:00 2001 From: kba Date: Tue, 14 Oct 2025 14:52:31 +0200 Subject: [PATCH 364/374] :memo: changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ad43c..dfd6868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Fixed: + + * Prevent OOM GPU error by avoiding loading the `region_fl` model, #199 + ## [0.6.0rc1] - 2025-10-10 Fixed: @@ -21,8 +25,7 @@ Fixed: * Dockerfile: fix CUDA installation (cuDNN contested between Torch and TF due to extra OCR) * OCR: re-instate missing methods and fix `utils_ocr` function calls * mbreorder/enhancement CLIs: missing imports - * :fire: writer: `SeparatorRegion` needs `SeparatorRegionType` (not `ImageRegionType`) -f458e3e + * :fire: writer: `SeparatorRegion` needs `SeparatorRegionType` (not `ImageRegionType`), f458e3e * tests: switch from `pytest-subtests` to `parametrize` so we can use `pytest-isolate` (so CUDA memory gets freed between tests if running on GPU) From c1f01588062714ba0c5146dc676c2dacade3e36f Mon Sep 17 00:00:00 2001 From: kba Date: Tue, 14 Oct 2025 14:53:15 +0200 Subject: [PATCH 365/374] :memo: changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd6868..636880f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Versioned according to [Semantic Versioning](http://semver.org/). Fixed: * Prevent OOM GPU error by avoiding loading the `region_fl` model, #199 + * XML output: encoding should be `utf-8`, not `utf8`, #196, #197 ## [0.6.0rc1] - 2025-10-10 From f485dd41819018a39960e45d5fd61c68d835cf1a Mon Sep 17 00:00:00 2001 From: kba Date: Tue, 14 Oct 2025 16:10:50 +0200 Subject: [PATCH 366/374] :package: v0.6.0rc2 --- CHANGELOG.md | 3 +++ src/eynollah/ocrd-tool.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 636880f..f84c153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +## [0.6.0rc2] - 2025-10-14 + Fixed: * Prevent OOM GPU error by avoiding loading the `region_fl` model, #199 @@ -295,6 +297,7 @@ Fixed: Initial release +[0.6.0rc2]: ../../compare/v0.6.0rc2...v0.6.0rc1 [0.6.0rc1]: ../../compare/v0.6.0rc1...v0.5.0 [0.5.0]: ../../compare/v0.5.0...v0.4.0 [0.4.0]: ../../compare/v0.4.0...v0.3.1 diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index 2ae4ead..f9c6f4d 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -1,5 +1,5 @@ { - "version": "0.6.0rc1", + "version": "0.6.0rc2", "git_url": "https://github.com/qurator-spk/eynollah", "dockerhub": "ocrd/eynollah", "tools": { From 948c8c3441f6dfa1f371e01a73f79ba957acd5c7 Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 15 Oct 2025 16:58:17 +0200 Subject: [PATCH 367/374] join_polygons: try to catch rare case of MultiPolygon --- src/eynollah/utils/contour.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 21068b3..f71bdc4 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -353,6 +353,8 @@ def join_polygons(polygons: Sequence[Polygon], scale=20) -> Polygon: bridgep = orient(LineString(nearest).buffer(max(1, scale/5), resolution=1), -1) polygons.append(bridgep) jointp = unary_union(polygons) + if jointp.geom_type == 'MultiPolygon': + jointp = unary_union(jointp.geoms) assert jointp.geom_type == 'Polygon', jointp.wkt # follow-up calculations will necessarily be integer; # so anticipate rounding here and then ensure validity From bd8c8bfeacbe6abb6e4217fe4008869af3ee97e9 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 16 Oct 2025 16:15:31 +0200 Subject: [PATCH 368/374] training: pin numpy to <1.24 as well --- train/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/train/requirements.txt b/train/requirements.txt index 2fb9908..63f3813 100644 --- a/train/requirements.txt +++ b/train/requirements.txt @@ -1,5 +1,6 @@ sacred seaborn +numpy <1.24.0 tqdm imutils scipy From d2f0a43088e31a8948b903b5b1de10cd695ce3ae Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 16 Oct 2025 20:46:03 +0200 Subject: [PATCH 369/374] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f84c153..249affa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Fixed: + + * `join_polygons` always returning Polygon, not MultiPolygon, #203 + ## [0.6.0rc2] - 2025-10-14 Fixed: From 2e0fb64dcb43894bdaf8df033471711fad2574f2 Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 16 Oct 2025 21:29:37 +0200 Subject: [PATCH 370/374] disable ruff check for training code for now --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 2945f6a..e7744a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,8 @@ source = ["eynollah"] [tool.ruff] line-length = 120 +# TODO: Reenable and fix after release v0.6.0 +exclude = ['src/eynollah/training'] [tool.ruff.lint] ignore = [ @@ -73,3 +75,4 @@ ignore = [ [tool.ruff.format] quote-style = "preserve" + From 2ac01ecaccbbddc36bc609fc9866c628e21b8ccc Mon Sep 17 00:00:00 2001 From: Robert Sachunsky Date: Wed, 15 Oct 2025 16:58:17 +0200 Subject: [PATCH 371/374] join_polygons: try to catch rare case of MultiPolygon --- src/eynollah/utils/contour.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py index 21068b3..f71bdc4 100644 --- a/src/eynollah/utils/contour.py +++ b/src/eynollah/utils/contour.py @@ -353,6 +353,8 @@ def join_polygons(polygons: Sequence[Polygon], scale=20) -> Polygon: bridgep = orient(LineString(nearest).buffer(max(1, scale/5), resolution=1), -1) polygons.append(bridgep) jointp = unary_union(polygons) + if jointp.geom_type == 'MultiPolygon': + jointp = unary_union(jointp.geoms) assert jointp.geom_type == 'Polygon', jointp.wkt # follow-up calculations will necessarily be integer; # so anticipate rounding here and then ensure validity From 46d25647f7d0cc1ea0354a9bd90f8e9479f32ffa Mon Sep 17 00:00:00 2001 From: kba Date: Thu, 16 Oct 2025 20:46:03 +0200 Subject: [PATCH 372/374] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f84c153..249affa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Fixed: + + * `join_polygons` always returning Polygon, not MultiPolygon, #203 + ## [0.6.0rc2] - 2025-10-14 Fixed: From ca8edb35e3cdaa789390835052c1781aa3331e63 Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 17 Oct 2025 10:35:13 +0200 Subject: [PATCH 373/374] :memo: changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10b3923..d8d7a6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +Added: + + * `eynollah-training` CLI and docs for training the models, #187, #193, https://github.com/qurator-spk/sbb_pixelwise_segmentation/tree/unifying-training-models + Fixed: * `join_polygons` always returning Polygon, not MultiPolygon, #203 From 38c028c6b500fcc7e2d5202f8930c38f74fc9bdc Mon Sep 17 00:00:00 2001 From: kba Date: Fri, 17 Oct 2025 10:36:30 +0200 Subject: [PATCH 374/374] :package: v0.6.0 --- CHANGELOG.md | 3 +++ src/eynollah/ocrd-tool.json | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8d7a6c..c2caaa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased +## [0.6.0] - 2025-10-17 + Added: * `eynollah-training` CLI and docs for training the models, #187, #193, https://github.com/qurator-spk/sbb_pixelwise_segmentation/tree/unifying-training-models @@ -305,6 +307,7 @@ Fixed: Initial release +[0.6.0]: ../../compare/v0.6.0...v0.6.0rc2 [0.6.0rc2]: ../../compare/v0.6.0rc2...v0.6.0rc1 [0.6.0rc1]: ../../compare/v0.6.0rc1...v0.5.0 [0.5.0]: ../../compare/v0.5.0...v0.4.0 diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json index f9c6f4d..dbbdc3b 100644 --- a/src/eynollah/ocrd-tool.json +++ b/src/eynollah/ocrd-tool.json @@ -1,5 +1,5 @@ { - "version": "0.6.0rc2", + "version": "0.6.0", "git_url": "https://github.com/qurator-spk/eynollah", "dockerhub": "ocrd/eynollah", "tools": {