diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..72b2c5a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,28 @@ +version: 2 + +jobs: + + build-python36: + docker: + - image: python:3.6 + steps: + - checkout + - restore_cache: + keys: + - model-cache + - run: make models + - save_cache: + key: model-cache + paths: + models_eynollah.tar.gz + models_eynollah + - run: make install + - run: make smoke-test + +workflows: + version: 2 + build: + jobs: + - build-python36 + #- build-python37 + #- build-python38 # no tensorflow for python 3.8 diff --git a/.gitignore b/.gitignore index c14b1f1..880a07d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ *.egg-info __pycache__ +sbb_newspapers_org_image/pylint.log +models_eynollah* +output.html diff --git a/Makefile b/Makefile index 9a96602..9855fa5 100644 --- a/Makefile +++ b/Makefile @@ -4,14 +4,26 @@ help: @echo "" @echo " Targets" @echo "" + @echo " models Download and extract models to $(PWD)/models_eynollah" @echo " install Install with pip" @echo " install-dev Install editable with pip" + @echo " test Run unit tests" @echo "" @echo " Variables" @echo "" # END-EVAL + +# Download and extract models to $(PWD)/models_eynollah +models: models_eynollah + +models_eynollah: models_eynollah.tar.gz + tar xf models_eynollah.tar.gz + +models_eynollah.tar.gz: + wget 'https://qurator-data.de/eynollah/models_eynollah.tar.gz' + # Install with pip install: pip install . @@ -19,3 +31,10 @@ install: # Install editable with pip install-dev: pip install -e . + +smoke-test: + eynollah -i tests/resources/kant_aufklaerung_1784_0020.tif -o . -m $(PWD)/models_eynollah + +# Run unit tests +test: + pytest tests diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..9bb2a15 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +pytest +black diff --git a/requirements.txt b/requirements.txt index 4016835..8368e37 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # ocrd includes opencv, numpy, shapely, click ocrd >= 2.20.1 -seaborn >= 0.11.0 keras >= 2.3.1 scikit-learn >= 0.23.2 -tensorflow >= 1.15, < 2 +tensorflow-gpu >= 1.15, < 2 imutils >= 0.5.3 +matplotlib diff --git a/sbb_newspapers_org_image/cli.py b/sbb_newspapers_org_image/cli.py new file mode 100644 index 0000000..03d0167 --- /dev/null +++ b/sbb_newspapers_org_image/cli.py @@ -0,0 +1,107 @@ +import click +from sbb_newspapers_org_image.eynollah import eynollah + + +@click.command() +@click.option( + "--image", "-i", help="image filename", type=click.Path(exists=True, dir_okay=False) +) +@click.option( + "--out", + "-o", + help="directory to write output xml data", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--model", + "-m", + help="directory of models", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_images", + "-si", + help="if a directory is given, images in documents will be cropped and saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_layout", + "-sl", + help="if a directory is given, plot of layout will be saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_deskewed", + "-sd", + help="if a directory is given, deskewed image will be saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_all", + "-sa", + help="if a directory is given, all plots needed for documentation will be saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--allow_enhancement", + "-ae", + is_flag=True, + help="if this parameter set to true, this tool would check that input image need resizing and enhancement or not. If so output of resized and enhanced image and corresponding layout data will be written in out directory", +) +@click.option( + "--curved_line", + "-cl", + is_flag=True, + help="if this parameter set to true, this tool will try to return contoure of textlines instead of rectabgle bounding box of textline. This should be taken into account that with this option the tool need more time to do process.", +) +@click.option( + "--full_layout", + "-fl", + is_flag=True, + help="if this parameter set to true, this tool will try to return all elements of layout.", +) +@click.option( + "--allow_scaling", + "-as", + is_flag=True, + help="if this parameter set to true, this tool would check the scale and if needed it will scale it to perform better layout detection", +) +@click.option( + "--headers_off", + "-ho", + is_flag=True, + help="if this parameter set to true, this tool would ignore headers role in reading order", +) +def main( + image, + out, + model, + save_images, + save_layout, + save_deskewed, + save_all, + allow_enhancement, + curved_line, + full_layout, + allow_scaling, + headers_off, +): + eynollah( + image, + None, + out, + model, + save_images, + save_layout, + save_deskewed, + save_all, + allow_enhancement, + curved_line, + full_layout, + allow_scaling, + headers_off, + ).run() + + +if __name__ == "__main__": + main() diff --git a/sbb_newspapers_org_image/eynollah.py b/sbb_newspapers_org_image/eynollah.py index 79f5993..3c38263 100644 --- a/sbb_newspapers_org_image/eynollah.py +++ b/sbb_newspapers_org_image/eynollah.py @@ -1,306 +1,196 @@ -#! /usr/bin/env python3 - -__version__ = '1.0' +""" +tool to extract table form data from alto xml data +""" +import gc +import math import os -os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' +import random import sys +import time +import warnings +from multiprocessing import Process, Queue, cpu_count +from sys import getsizeof + import cv2 import numpy as np import matplotlib.pyplot as plt -import seaborn as sns -from sys import getsizeof -import random -from tqdm import tqdm -from keras.models import model_from_json -from keras.models import load_model -import math -from shapely import geometry -from sklearn.cluster import KMeans -import gc + +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" +stderr = sys.stderr +sys.stderr = open(os.devnull, "w") from keras import backend as K +from keras.models import load_model +sys.stderr = stderr import tensorflow as tf -tf.get_logger().setLevel('ERROR') +tf.get_logger().setLevel("ERROR") +warnings.filterwarnings("ignore") + from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d -import xml.etree.ElementTree as ET -import warnings -import click -import time -from multiprocessing import Process, Queue, cpu_count +from shapely import geometry +import xml.etree.ElementTree as ET#from lxml import etree as ET from matplotlib import pyplot, transforms import matplotlib.patches as mpatches import imutils -warnings.filterwarnings('ignore') -# -__doc__ = \ - """ - tool to extract table form data from alto xml data - """ +from .utils.contour import ( + contours_in_same_horizon, + filter_contours_area_of_image_interiors, + filter_contours_area_of_image_tables, + filter_contours_area_of_image, + find_contours_mean_y_diff, + find_features_of_contours, + find_new_features_of_contoures, + get_text_region_boxes_by_given_contours, + get_textregion_contours_in_org_image, + return_bonding_box_of_contours, + return_contours_of_image, + return_contours_of_interested_region, + return_contours_of_interested_region_and_bounding_box, + return_contours_of_interested_region_by_min_size, + return_contours_of_interested_textline, + return_parent_contours, + return_contours_of_interested_region_by_size, +) + +from .utils.rotate import ( + rotate_image, + rotate_max_area, + rotate_max_area_new, + rotatedRectWithMaxArea, + rotation_image_new, + rotation_not_90_func, + rotation_not_90_func_full_layout, + rotyate_image_different, +) + +from .utils.separate_lines import ( + seperate_lines, + seperate_lines_new_inside_teils, + seperate_lines_new_inside_teils2, + seperate_lines_vertical, + seperate_lines_vertical_cont, + textline_contours_postprocessing, + seperate_lines_new2, + return_deskew_slop, +) + +from .utils.drop_capitals import ( + adhere_drop_capital_region_into_cprresponding_textline, + filter_small_drop_capitals_from_no_patch_layout +) + +from .utils.marginals import get_marginals + +from .utils.resize import resize_image + +from .utils import ( + boosting_headers_by_longshot_region_segmentation, + crop_image_inside_box, + find_features_of_lines, + find_num_col, + find_num_col_by_vertical_lines, + find_num_col_deskew, + find_num_col_only_image, + isNaN, + otsu_copy, + otsu_copy_binary, + return_hor_spliter_by_index_for_without_verticals, + delete_seperator_around, + return_regions_without_seperators, + put_drop_out_from_only_drop_model, + putt_bb_of_drop_capitals_of_model_in_patches_in_layout, + check_any_text_region_in_model_one_is_main_or_header, + small_textlines_to_parent_adherence2, + order_and_id_of_texts, + order_of_regions, + implent_law_head_main_not_parallel, + return_hor_spliter_by_index, + combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new, + return_points_with_boundies, + find_number_of_columns_in_document, + return_boxes_of_images_by_order_of_reading_new, +) + + +SLOPE_THRESHOLD = 0.13 class eynollah: - def __init__(self, image_dir, f_name,dir_out, dir_models,dir_of_cropped_images=None,dir_of_layout=None,dir_of_deskewed=None,dir_of_all=None,allow_enhancement='false',curved_line='false',full_layout='false',allow_scaling='false',headers_off='false'): + def __init__( + self, + image_dir, + f_name, + dir_out, + dir_models, + dir_of_cropped_images=None, + dir_of_layout=None, + dir_of_deskewed=None, + dir_of_all=None, + allow_enhancement=False, + curved_line=False, + full_layout=False, + allow_scaling=False, + headers_off=False + ): self.image_dir = image_dir # XXX This does not seem to be a directory as the name suggests, but a file self.dir_out = dir_out self.f_name = f_name - self.dir_of_cropped_images=dir_of_cropped_images - self.allow_enhancement=allow_enhancement - self.curved_line=curved_line - self.full_layout=full_layout - self.allow_scaling=allow_scaling - self.dir_of_layout=dir_of_layout - self.headers_off=headers_off - self.dir_of_deskewed=dir_of_deskewed - self.dir_of_all=dir_of_all + self.dir_of_cropped_images = dir_of_cropped_images + self.allow_enhancement = allow_enhancement + self.curved_line = curved_line + self.full_layout = full_layout + self.allow_scaling = allow_scaling + self.dir_of_layout = dir_of_layout + self.headers_off = headers_off + self.dir_of_deskewed = dir_of_deskewed + self.dir_of_all = dir_of_all if self.f_name is None: try: - self.f_name = image_dir.split('/')[len(image_dir.split('/')) - 1] - self.f_name = self.f_name.split('.')[0] + self.f_name = image_dir.split("/")[len(image_dir.split("/")) - 1] + self.f_name = self.f_name.split(".")[0] except: - self.f_name = self.f_name.split('.')[0] + self.f_name = self.f_name.split(".")[0] self.dir_models = dir_models self.kernel = np.ones((5, 5), np.uint8) - - self.model_dir_of_enhancemnet=dir_models +'/model_enhancement.h5' - self.model_dir_of_col_classifier=dir_models +'/model_scale_classifier.h5' - self.model_region_dir_p = dir_models +'/model_main_covid19_lr5-5_scale_1_1_great.h5'#dir_models +'/model_main_covid_19_many_scalin_down_lr5-5_the_best.h5'#'/model_main_covid19_lr5-5_scale_1_1_great.h5'#'/model_main_scale_1_1und_1_2_corona_great.h5' - #self.model_region_dir_p_ens = dir_models +'/model_ensemble_s.h5'#'/model_main_covid19_lr5-5_scale_1_1_great.h5'#'/model_main_scale_1_1und_1_2_corona_great.h5' - self.model_region_dir_p2 = dir_models +'/model_main_home_corona3_rot.h5' - - self.model_region_dir_fully_np = dir_models + '/model_no_patches_class0_30eopch.h5' - self.model_region_dir_fully = dir_models + '/model_3up_new_good_no_augmentation.h5'#"model_3col_p_soft_10_less_aug_binarization_only.h5" - - self.model_page_dir = dir_models + '/model_page_mixed_best.h5' - self.model_region_dir_p_ens =dir_models +'/model_ensemble_s.h5'#dir_models +'/model_main_covid_19_many_scalin_down_lr5-5_the_best.h5' #dir_models +'/model_ensemble_s.h5' - ###self.model_region_dir_p = dir_models +'/model_layout_newspapers.h5'#'/model_ensemble_s.h5'#'/model_layout_newspapers.h5'#'/model_ensemble_s.h5'#'/model_main_home_5_soft_new.h5'#'/model_home_soft_5_all_data.h5' #'/model_main_office_long_soft.h5'#'/model_20_cat_main.h5' - self.model_textline_dir = dir_models + '/model_textline_newspapers.h5'#'/model_hor_ver_home_trextline_very_good.h5'# '/model_hor_ver_1_great.h5'#'/model_curved_office_works_great.h5' - - - def filter_contours_area_of_image_tables(self,image,contours,hirarchy,max_area,min_area): - found_polygons_early = list() - - jv=0 - for c in contours: - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - #area = cv2.contourArea(c) - area = polygon.area - ##print(np.prod(thresh.shape[:2])) - # Check that polygon has area greater than minimal area - #print(hirarchy[0][jv][3],hirarchy ) - if area >=min_area*np.prod(image.shape[:2]) and area <=max_area*np.prod(image.shape[:2]):#and hirarchy[0][jv][3]==-1 : - #print(c[0][0][1]) - found_polygons_early.append( - np.array( [ [point] for point in polygon.exterior.coords] , dtype=np.int32) ) - jv+=1 - return found_polygons_early - def find_polygons_size_filter(self, contours, median_area, scaler_up=1.2, scaler_down=0.8): - found_polygons_early = list() - - for c in contours: - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - area = polygon.area - # Check that polygon has area greater than minimal area - if area >= median_area * scaler_down and area <= median_area * scaler_up: - found_polygons_early.append( - np.array([point for point in polygon.exterior.coords], dtype=np.uint)) - return found_polygons_early - - def filter_contours_area_of_image(self, image, contours, hirarchy, max_area, min_area): - found_polygons_early = list() - - jv = 0 - for c in contours: - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - area = polygon.area - if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod( - image.shape[:2]) and hirarchy[0][jv][3] == -1 : # and hirarchy[0][jv][3]==-1 : - found_polygons_early.append( - np.array([ [point] for point in polygon.exterior.coords], dtype=np.uint)) - jv += 1 - return found_polygons_early - - def filter_contours_area_of_image_interiors(self, image, contours, hirarchy, max_area, min_area): - found_polygons_early = list() - - jv = 0 - for c in contours: - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - area = polygon.area - if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and \ - hirarchy[0][jv][3] != -1: - # print(c[0][0][1]) - found_polygons_early.append( - np.array([point for point in polygon.exterior.coords], dtype=np.uint)) - jv += 1 - return found_polygons_early - - def resize_image(self, img_in, input_height, input_width): - return cv2.resize(img_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) - - def resize_ann(self, seg_in, input_height, input_width): - return cv2.resize(seg_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) - - def rotatedRectWithMaxArea(self,w, h, angle): - if w <= 0 or h <= 0: - return 0,0 - - width_is_longer = w >= h - side_long, side_short = (w,h) if width_is_longer else (h,w) - - # since the solutions for angle, -angle and 180-angle are all the same, - # if suffices to look at the first quadrant and the absolute values of sin,cos: - sin_a, cos_a = abs(math.sin(angle)), abs(math.cos(angle)) - if side_short <= 2.*sin_a*cos_a*side_long or abs(sin_a-cos_a) < 1e-10: - # half constrained case: two crop corners touch the longer side, - # the other two corners are on the mid-line parallel to the longer line - x = 0.5*side_short - wr,hr = (x/sin_a,x/cos_a) if width_is_longer else (x/cos_a,x/sin_a) - else: - # fully constrained case: crop touches all 4 sides - cos_2a = cos_a*cos_a - sin_a*sin_a - wr,hr = (w*cos_a - h*sin_a)/cos_2a, (h*cos_a - w*sin_a)/cos_2a - - return wr,hr - - def get_one_hot(self, seg, input_height, input_width, n_classes): - seg = seg[:, :, 0] - seg_f = np.zeros((input_height, input_width, n_classes)) - for j in range(n_classes): - seg_f[:, :, j] = (seg == j).astype(int) - return seg_f - - - def color_images(self, seg, n_classes): - ann_u = range(n_classes) - if len(np.shape(seg)) == 3: - seg = seg[:, :, 0] - - seg_img = np.zeros((np.shape(seg)[0], np.shape(seg)[1], 3)).astype(np.uint8) - colors = sns.color_palette("hls", n_classes) - - for c in ann_u: - c = int(c) - segl = (seg == c) - seg_img[:, :, 0] = segl * c - seg_img[:, :, 1] = segl * c - seg_img[:, :, 2] = segl * c - return seg_img - - def color_images_diva(self, seg, n_classes): - ann_u = range(n_classes) - if len(np.shape(seg)) == 3: - seg = seg[:, :, 0] - - seg_img = np.zeros((np.shape(seg)[0], np.shape(seg)[1], 3)).astype(float) - # colors=sns.color_palette("hls", n_classes) - colors = [[1, 0, 0], [8, 0, 0], [2, 0, 0], [4, 0, 0]] - - for c in ann_u: - c = int(c) - segl = (seg == c) - seg_img[:, :, 0][seg == c] = colors[c][0] # segl*(colors[c][0]) - seg_img[:, :, 1][seg == c] = colors[c][1] # seg_img[:,:,1]=segl*(colors[c][1]) - seg_img[:, :, 2][seg == c] = colors[c][2] # seg_img[:,:,2]=segl*(colors[c][2]) - return seg_img - - def rotate_image(self, img_patch, slope): - (h, w) = img_patch.shape[:2] - center = (w // 2, h // 2) - M = cv2.getRotationMatrix2D(center, slope, 1.0) - return cv2.warpAffine(img_patch, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) - - def rotyate_image_different(self,img,slope): - #img = cv2.imread('images/input.jpg') - num_rows, num_cols = img.shape[:2] - - rotation_matrix = cv2.getRotationMatrix2D((num_cols/2, num_rows/2), slope, 1) - img_rotation = cv2.warpAffine(img, rotation_matrix, (num_cols, num_rows) ) - return img_rotation - - def cleaning_probs(self, probs: np.ndarray, sigma: float) -> np.ndarray: - # Smooth - if sigma > 0.: - return cv2.GaussianBlur(probs, (int(3 * sigma) * 2 + 1, int(3 * sigma) * 2 + 1), sigma) - elif sigma == 0.: - return cv2.fastNlMeansDenoising((probs * 255).astype(np.uint8), h=20) / 255 - else: # Negative sigma, do not do anything - return probs - - def crop_image_inside_box(self, box, img_org_copy): - image_box = img_org_copy[box[1]:box[1] + box[3], box[0]:box[0] + box[2]] - return image_box, [box[1], box[1] + box[3], box[0], box[0] + box[2]] - - def otsu_copy(self, img): - img_r = np.zeros(img.shape) - img1 = img[:, :, 0] - img2 = img[:, :, 1] - img3 = img[:, :, 2] - # print(img.min()) - # print(img[:,:,0].min()) - # blur = cv2.GaussianBlur(img,(5,5)) - # ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) - retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) - retval2, threshold2 = cv2.threshold(img2, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) - retval3, threshold3 = cv2.threshold(img3, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) - - img_r[:, :, 0] = threshold1 - img_r[:, :, 1] = threshold1 - img_r[:, :, 2] = threshold1 - return img_r - - def otsu_copy_binary(self,img): - img_r=np.zeros((img.shape[0],img.shape[1],3)) - img1=img[:,:,0] - - retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU) - + self.model_dir_of_enhancemnet = dir_models + "/model_enhancement.h5" + self.model_dir_of_col_classifier = dir_models + "/model_scale_classifier.h5" + self.model_region_dir_p = dir_models + "/model_main_covid19_lr5-5_scale_1_1_great.h5" # dir_models +'/model_main_covid_19_many_scalin_down_lr5-5_the_best.h5'#'/model_main_covid19_lr5-5_scale_1_1_great.h5'#'/model_main_scale_1_1und_1_2_corona_great.h5' + # self.model_region_dir_p_ens = dir_models +'/model_ensemble_s.h5'#'/model_main_covid19_lr5-5_scale_1_1_great.h5'#'/model_main_scale_1_1und_1_2_corona_great.h5' + self.model_region_dir_p2 = dir_models + "/model_main_home_corona3_rot.h5" - img_r[:,:,0]=threshold1 - img_r[:,:,1]=threshold1 - img_r[:,:,2]=threshold1 + self.model_region_dir_fully_np = dir_models + "/model_no_patches_class0_30eopch.h5" + self.model_region_dir_fully = dir_models + "/model_3up_new_good_no_augmentation.h5" # "model_3col_p_soft_10_less_aug_binarization_only.h5" + + self.model_page_dir = dir_models + "/model_page_mixed_best.h5" + self.model_region_dir_p_ens = dir_models + "/model_ensemble_s.h5" # dir_models +'/model_main_covid_19_many_scalin_down_lr5-5_the_best.h5' #dir_models +'/model_ensemble_s.h5' + ###self.model_region_dir_p = dir_models +'/model_layout_newspapers.h5'#'/model_ensemble_s.h5'#'/model_layout_newspapers.h5'#'/model_ensemble_s.h5'#'/model_main_home_5_soft_new.h5'#'/model_home_soft_5_all_data.h5' #'/model_main_office_long_soft.h5'#'/model_20_cat_main.h5' + self.model_textline_dir = dir_models + "/model_textline_newspapers.h5" #'/model_hor_ver_home_trextline_very_good.h5'# '/model_hor_ver_1_great.h5'#'/model_curved_office_works_great.h5' + + def predict_enhancement(self, img): + model_enhancement, session_enhancemnet = self.start_new_session_and_model(self.model_dir_of_enhancemnet) - img_r=img_r/float(np.max(img_r))*255 - return img_r - - def predict_enhancement(self,img): - model_enhancement,session_enhancemnet=self.start_new_session_and_model(self.model_dir_of_enhancemnet) - img_height_model = model_enhancement.layers[len(model_enhancement.layers) - 1].output_shape[1] img_width_model = model_enhancement.layers[len(model_enhancement.layers) - 1].output_shape[2] - #n_classes = model_enhancement.layers[len(model_enhancement.layers) - 1].output_shape[3] - if img.shape[0] img_h: index_y_u = img_h index_y_d = img_h - img_height_model - - img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - label_p_pred = model_enhancement.predict( - img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2])) - - seg =label_p_pred[0,:,:,:] - seg=seg*255 - - - if i==0 and j==0: - seg = seg[0:seg.shape[0] - margin, 0:seg.shape[1] - margin] - prediction_true[index_y_d + 0:index_y_u - margin, index_x_d + 0:index_x_u - margin, - :] = seg - - elif i==nxf-1 and j==nyf-1: - seg = seg[margin:seg.shape[0] - 0, margin:seg.shape[1] - 0] - prediction_true[index_y_d + margin:index_y_u - 0, index_x_d + margin:index_x_u - 0, - :] = seg - - elif i==0 and j==nyf-1: - seg = seg[margin:seg.shape[0] - 0, 0:seg.shape[1] - margin] - - prediction_true[index_y_d + margin:index_y_u - 0, index_x_d + 0:index_x_u - margin, - :] = seg - - elif i==nxf-1 and j==0: - seg = seg[0:seg.shape[0] - margin, margin:seg.shape[1] - 0] - - prediction_true[index_y_d + 0:index_y_u - margin, index_x_d + margin:index_x_u - 0, - :] = seg - - elif i==0 and j!=0 and j!=nyf-1: - seg = seg[margin:seg.shape[0] - margin, 0:seg.shape[1] - margin] - - prediction_true[index_y_d + margin:index_y_u - margin, index_x_d + 0:index_x_u - margin, - :] = seg - - elif i==nxf-1 and j!=0 and j!=nyf-1: - seg = seg[margin:seg.shape[0] - margin, margin:seg.shape[1] - 0] - - prediction_true[index_y_d + margin:index_y_u - margin, index_x_d + margin:index_x_u - 0, - :] = seg - - elif i!=0 and i!=nxf-1 and j==0: - seg = seg[0:seg.shape[0] - margin, margin:seg.shape[1] - margin] - prediction_true[index_y_d + 0:index_y_u - margin, index_x_d + margin:index_x_u - margin, - :] = seg - - elif i!=0 and i!=nxf-1 and j==nyf-1: - seg = seg[margin:seg.shape[0] - 0, margin:seg.shape[1] - margin] - - prediction_true[index_y_d + margin:index_y_u - 0, index_x_d + margin:index_x_u - margin, - :] = seg + label_p_pred = model_enhancement.predict(img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2])) + + seg = label_p_pred[0, :, :, :] + seg = seg * 255 + + if i == 0 and j == 0: + seg = seg[0 : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg + + elif i == nxf - 1 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - 0] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0, :] = seg + + elif i == 0 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, 0 : seg.shape[1] - margin] + + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin, :] = seg + + elif i == nxf - 1 and j == 0: + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - 0] + + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg + + elif i == 0 and j != 0 and j != nyf - 1: + seg = seg[margin : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg + + elif i == nxf - 1 and j != 0 and j != nyf - 1: + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - 0] + + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg + + elif i != 0 and i != nxf - 1 and j == 0: + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - margin] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg + + elif i != 0 and i != nxf - 1 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - margin] + + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin, :] = seg else: - seg = seg[margin:seg.shape[0] - margin, margin:seg.shape[1] - margin] + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - margin] - prediction_true[index_y_d + margin:index_y_u - margin, index_x_d + margin:index_x_u - margin, - :] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg prediction_true = prediction_true.astype(int) - + del model_enhancement del session_enhancemnet - return prediction_true - + def check_dpi(self): - self.dpi=os.popen('identify -format "%x " '+self.image_dir).read() - self.dpi=int(float(self.dpi)) - - - def resize_image_with_column_classifier(self,is_image_enhanced): - self.check_dpi() - img=cv2.imread(self.image_dir) - img=img.astype(np.uint8) + dpi = os.popen('identify -format "%x " ' + self.image_dir).read() + return int(float(dpi)) + + def resize_image_with_column_classifier(self, is_image_enhanced): + dpi = self.check_dpi() + img = cv2.imread(self.image_dir) + img = img.astype(np.uint8) - _,page_coord=self.early_page_for_num_of_column_classification() + _, page_coord = self.early_page_for_num_of_column_classification() model_num_classifier, session_col_classifier = self.start_new_session_and_model(self.model_dir_of_col_classifier) - - img_1ch=cv2.imread(self.image_dir,0) - - width_early=img_1ch.shape[1] - - img_1ch=img_1ch[page_coord[0]:page_coord[1],page_coord[2]:page_coord[3]] - - #plt.imshow(img_1ch) - #plt.show() - img_1ch=img_1ch/255. - - img_1ch=cv2.resize( img_1ch, ( 448,448) ,interpolation=cv2.INTER_NEAREST) - - img_in=np.zeros((1,img_1ch.shape[0],img_1ch.shape[1],3)) - img_in[0,:,:,0]=img_1ch[:,:] - img_in[0,:,:,1]=img_1ch[:,:] - img_in[0,:,:,2]=img_1ch[:,:] - - label_p_pred=model_num_classifier.predict(img_in) - num_col=np.argmax(label_p_pred[0])+1 - - print(num_col,label_p_pred,'num_col_classifier') - + + img_1ch = cv2.imread(self.image_dir, 0) + + width_early = img_1ch.shape[1] + + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + # plt.imshow(img_1ch) + # plt.show() + img_1ch = img_1ch / 255.0 + + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = model_num_classifier.predict(img_in) + num_col = np.argmax(label_p_pred[0]) + 1 + + print(num_col, label_p_pred, "num_col_classifier") + session_col_classifier.close() del model_num_classifier del session_col_classifier - + K.clear_session() gc.collect() - - #sys.exit() - if num_col==1 and width_early<1100: - img_w_new=2000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2000) - - elif num_col==1 and width_early>=2500: - img_w_new=2000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2000) - elif num_col==1 and width_early>=1100 and width_early<2500: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - elif num_col==2 and width_early<2000: - img_w_new=2400 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2400) - - elif num_col==2 and width_early>=3500: - img_w_new=2400 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2400) - - elif num_col==2 and width_early>=2000 and width_early<3500: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - elif num_col==3 and width_early<2000: - img_w_new=3000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 3000) - - elif num_col==3 and width_early>=4000: - img_w_new=3000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 3000) - - elif num_col==3 and width_early>=2000 and width_early<4000: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - - elif num_col==4 and width_early<2500: - img_w_new=4000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 4000) - - elif num_col==4 and width_early>=5000: - img_w_new=4000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 4000) - - elif num_col==4 and width_early>=2500 and width_early<5000: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - - - elif num_col==5 and width_early<3700: - img_w_new=5000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 5000) - - elif num_col==5 and width_early>=7000: - img_w_new=5000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 5000) - - elif num_col==5 and width_early>=3700 and width_early<7000: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - - - elif num_col==6 and width_early<4500: - img_w_new=6500#5400 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 6500) - + # sys.exit() + if num_col == 1 and width_early < 1100: + img_w_new = 2000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2000) + + elif num_col == 1 and width_early >= 2500: + img_w_new = 2000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2000) + elif num_col == 1 and width_early >= 1100 and width_early < 2500: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 2 and width_early < 2000: + img_w_new = 2400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2400) + + elif num_col == 2 and width_early >= 3500: + img_w_new = 2400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2400) + + elif num_col == 2 and width_early >= 2000 and width_early < 3500: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 3 and width_early < 2000: + img_w_new = 3000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 3000) + + elif num_col == 3 and width_early >= 4000: + img_w_new = 3000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 3000) + + elif num_col == 3 and width_early >= 2000 and width_early < 4000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 4 and width_early < 2500: + img_w_new = 4000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 4000) + + elif num_col == 4 and width_early >= 5000: + img_w_new = 4000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 4000) + + elif num_col == 4 and width_early >= 2500 and width_early < 5000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 5 and width_early < 3700: + img_w_new = 5000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 5000) + + elif num_col == 5 and width_early >= 7000: + img_w_new = 5000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 5000) + + elif num_col == 5 and width_early >= 3700 and width_early < 7000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 6 and width_early < 4500: + img_w_new = 6500 # 5400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 6500) + else: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) - if label_p_pred[0][int(num_col-1)]<0.9 and img_w_newimg.shape[1]: - img_new=self.predict_enhancement(img_new) - is_image_enhanced=True - - return img,img_new,is_image_enhanced - - def resize_and_enhance_image_with_column_classifier(self,is_image_enhanced): - self.check_dpi() - img=cv2.imread(self.image_dir) - - img=img.astype(np.uint8) - - _,page_coord=self.early_page_for_num_of_column_classification() + if img_new.shape[1] > img.shape[1]: + img_new = self.predict_enhancement(img_new) + is_image_enhanced = True + + return img, img_new, is_image_enhanced + + def resize_and_enhance_image_with_column_classifier(self, is_image_enhanced): + dpi = self.check_dpi() + img = cv2.imread(self.image_dir) + + img = img.astype(np.uint8) + + _, page_coord = self.early_page_for_num_of_column_classification() model_num_classifier, session_col_classifier = self.start_new_session_and_model(self.model_dir_of_col_classifier) - - img_1ch=cv2.imread(self.image_dir,0) - img_1ch=img_1ch.astype(np.uint8) - - width_early=img_1ch.shape[1] - - img_1ch=img_1ch[page_coord[0]:page_coord[1],page_coord[2]:page_coord[3]] - - #plt.imshow(img_1ch) - #plt.show() - img_1ch=img_1ch/255. - - img_1ch=cv2.resize( img_1ch, ( 448,448) ,interpolation=cv2.INTER_NEAREST) - - img_in=np.zeros((1,img_1ch.shape[0],img_1ch.shape[1],3)) - img_in[0,:,:,0]=img_1ch[:,:] - img_in[0,:,:,1]=img_1ch[:,:] - img_in[0,:,:,2]=img_1ch[:,:] - - - #plt.imshow(img_in[0,:,:,:]) - #plt.show() - - label_p_pred=model_num_classifier.predict(img_in) - num_col=np.argmax(label_p_pred[0])+1 - - print(num_col,label_p_pred,'num_col_classifier') - + + img_1ch = cv2.imread(self.image_dir, 0) + img_1ch = img_1ch.astype(np.uint8) + + width_early = img_1ch.shape[1] + + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + # plt.imshow(img_1ch) + # plt.show() + img_1ch = img_1ch / 255.0 + + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + # plt.imshow(img_in[0,:,:,:]) + # plt.show() + + label_p_pred = model_num_classifier.predict(img_in) + num_col = np.argmax(label_p_pred[0]) + 1 + + print(num_col, label_p_pred, "num_col_classifier") + session_col_classifier.close() del model_num_classifier del session_col_classifier del img_in del img_1ch del page_coord - + K.clear_session() gc.collect() - - - print(self.dpi) - - if self.dpi<298: - #sys.exit() - if num_col==1 and width_early<1100: - img_w_new=2000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2000) - - elif num_col==1 and width_early>=2500: - img_w_new=2000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2000) - elif num_col==1 and width_early>=1100 and width_early<2500: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - elif num_col==2 and width_early<2000: - img_w_new=2400 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2400) - - elif num_col==2 and width_early>=3500: - img_w_new=2400 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 2400) - - elif num_col==2 and width_early>=2000 and width_early<3500: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - + print(dpi) + + if dpi < 298: + + # sys.exit() + if num_col == 1 and width_early < 1100: + img_w_new = 2000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2000) + + elif num_col == 1 and width_early >= 2500: + img_w_new = 2000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2000) + elif num_col == 1 and width_early >= 1100 and width_early < 2500: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 2 and width_early < 2000: + img_w_new = 2400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2400) + + elif num_col == 2 and width_early >= 3500: + img_w_new = 2400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2400) + + elif num_col == 2 and width_early >= 2000 and width_early < 3500: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 3 and width_early < 2000: + img_w_new = 3000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 3000) + + elif num_col == 3 and width_early >= 4000: + img_w_new = 3000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 3000) + + elif num_col == 3 and width_early >= 2000 and width_early < 4000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 4 and width_early < 2500: + img_w_new = 4000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 4000) + + elif num_col == 4 and width_early >= 5000: + img_w_new = 4000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 4000) + + elif num_col == 4 and width_early >= 2500 and width_early < 5000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 5 and width_early < 3700: + img_w_new = 5000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 5000) + + elif num_col == 5 and width_early >= 7000: + img_w_new = 5000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 5000) + + elif num_col == 5 and width_early >= 3700 and width_early < 7000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + elif num_col == 6 and width_early < 4500: + img_w_new = 6500 # 5400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 6500) - elif num_col==3 and width_early<2000: - img_w_new=3000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 3000) - - elif num_col==3 and width_early>=4000: - img_w_new=3000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 3000) - - elif num_col==3 and width_early>=2000 and width_early<4000: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - - elif num_col==4 and width_early<2500: - img_w_new=4000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 4000) - - elif num_col==4 and width_early>=5000: - img_w_new=4000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 4000) - - elif num_col==4 and width_early>=2500 and width_early<5000: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - - - elif num_col==5 and width_early<3700: - img_w_new=5000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 5000) - - elif num_col==5 and width_early>=7000: - img_w_new=5000 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 5000) - - elif num_col==5 and width_early>=3700 and width_early<7000: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - - - elif num_col==6 and width_early<4500: - img_w_new=6500#5400 - img_h_new=int(img.shape[0]/float(img.shape[1]) * 6500) - else: - img_w_new=width_early - img_h_new=int(img.shape[0]/float(img.shape[1]) * width_early) - - + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) - if label_p_pred[0][int(num_col-1)]<0.9 and img_w_new=img.shape[1]: img_h_new=3000 img_w_new=int(img.shape[1]/float(img.shape[0]) * 3000) - img_new=self.resize_image(img,img_h_new,img_w_new) + img_new=resize_image(img,img_h_new,img_w_new) image_res=self.predict_enhancement(img_new) #cv2.imwrite(os.path.join(self.dir_out, self.f_name) + ".tif",self.image) #self.image=self.image.astype(np.uint16) @@ -707,123 +564,45 @@ class eynollah: else: is_image_enhanced=False image_res=np.copy(img) - + """ - is_image_enhanced=False - num_column_is_classified=True - image_res=np.copy(img) - - + is_image_enhanced = False + num_column_is_classified = True + image_res = np.copy(img) - - return is_image_enhanced,img,image_res,num_col,num_column_is_classified - def resize_and_enhance_image(self,is_image_enhanced): - self.check_dpi() - img=cv2.imread(self.image_dir) - img=img.astype(np.uint8) - #sys.exit() - - - print(self.dpi) - - if self.dpi<298: - if img.shape[0]<1000: - img_h_new=int(img.shape[0]*3) - img_w_new=int(img.shape[1]*3) - if img_h_new<2800: - img_h_new=3000 - img_w_new=int(img.shape[1]/float(img.shape[0]) * 3000) - elif img.shape[0]>=1000 and img.shape[0]<2000: - img_h_new=int(img.shape[0]*2) - img_w_new=int(img.shape[1]*2) - if img_h_new<2800: - img_h_new=3000 - img_w_new=int(img.shape[1]/float(img.shape[0]) * 3000) - else: - img_h_new=int(img.shape[0]*1.5) - img_w_new=int(img.shape[1]*1.5) - img_new=self.resize_image(img,img_h_new,img_w_new) - image_res=self.predict_enhancement(img_new) - #cv2.imwrite(os.path.join(self.dir_out, self.f_name) + ".tif",self.image) - #self.image=self.image.astype(np.uint16) - - #self.scale_x=1 - #self.scale_y=1 - #self.height_org = self.image.shape[0] - #self.width_org = self.image.shape[1] - is_image_enhanced=True - else: - is_image_enhanced=False - image_res=np.copy(img) - - return is_image_enhanced,img,image_res - - - def resize_and_enhance_image_new(self,is_image_enhanced): - #self.check_dpi() - img=cv2.imread(self.image_dir) - img=img.astype(np.uint8) - #sys.exit() + return is_image_enhanced, img, image_res, num_col, num_column_is_classified - image_res=np.copy(img) - - - return is_image_enhanced,img,image_res - - - - - def get_image_and_scales(self,img_org,img_res,scale): + def get_image_and_scales(self, img_org, img_res, scale): self.image = np.copy(img_res) - self.image_org=np.copy(img_org) + self.image_org = np.copy(img_org) self.height_org = self.image.shape[0] self.width_org = self.image.shape[1] - - - - self.img_hight_int =int(self.image.shape[0]*scale) - self.img_width_int = int(self.image.shape[1]*scale) + + self.img_hight_int = int(self.image.shape[0] * scale) + self.img_width_int = int(self.image.shape[1] * scale) self.scale_y = self.img_hight_int / float(self.image.shape[0]) self.scale_x = self.img_width_int / float(self.image.shape[1]) - self.image = self.resize_image(self.image, self.img_hight_int, self.img_width_int) + self.image = resize_image(self.image, self.img_hight_int, self.img_width_int) del img_res del img_org - def get_image_and_scales_after_enhancing(self,img_org,img_res): - - #self.image = cv2.imread(self.image_dir) - - self.image=np.copy(img_res) - self.image=self.image.astype(np.uint8) - self.image_org=np.copy(img_org) + + def get_image_and_scales_after_enhancing(self, img_org, img_res): + + # self.image = cv2.imread(self.image_dir) + + self.image = np.copy(img_res) + self.image = self.image.astype(np.uint8) + self.image_org = np.copy(img_org) self.height_org = self.image_org.shape[0] self.width_org = self.image_org.shape[1] - - self.scale_y = img_res.shape[0] / float(self.image_org.shape[0]) - self.scale_x = img_res.shape[1]/ float(self.image_org.shape[1]) - + self.scale_x = img_res.shape[1] / float(self.image_org.shape[1]) + del img_org del img_res - - - def get_image_and_scales_deskewd(self,img_deskewd): - - self.image = img_deskewd - self.image_org=np.copy(self.image) - self.height_org = self.image.shape[0] - self.width_org = self.image.shape[1] - - - self.img_hight_int =int(self.image.shape[0]*1) - self.img_width_int = int(self.image.shape[1]*1) - self.scale_y = self.img_hight_int / float(self.image.shape[0]) - self.scale_x = self.img_width_int / float(self.image.shape[1]) - - self.image = self.resize_image(self.image, self.img_hight_int, self.img_width_int) - def start_new_session_and_model(self, model_dir): config = tf.ConfigProto() config.gpu_options.allow_growth = True @@ -832,216 +611,38 @@ class eynollah: model = load_model(model_dir, compile=False) return model, session - - def return_bonding_box_of_contours(self,cnts): - boxes_tot=[] - for i in range(len(cnts)): - x,y,w,h = cv2.boundingRect(cnts[i]) - - box=[x,y,w,h] - boxes_tot.append(box) - return boxes_tot - - def find_features_of_lines(self,contours_main): - - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - slope_lines=[] - - for kk in range(len(contours_main)): - [vx,vy,x,y] = cv2.fitLine(contours_main[kk], cv2.DIST_L2,0,0.01,0.01) - slope_lines.append( ( (vy/vx)/np.pi*180 )[0] ) - - slope_lines_org=slope_lines - slope_lines=np.array(slope_lines) - slope_lines[(slope_lines<10) & (slope_lines>-10)]=0 - - slope_lines[(slope_lines<-200) | (slope_lines>200)]=1 - slope_lines[ (slope_lines!=0) & (slope_lines!=1)]=2 - - dis_x=np.abs(x_max_main-x_min_main) - return slope_lines,dis_x, x_min_main ,x_max_main ,np.array(cy_main),np.array(slope_lines_org),y_min_main ,y_max_main,np.array(cx_main) - - def return_parent_contours(self,contours,hierarchy): - contours_parent=[ contours[i] for i in range(len(contours) ) if hierarchy[0][i][3]==-1 ] - return contours_parent - - def isNaN(self,num): - return num != num - def early_deskewing_slope_calculation_based_on_lines(self,region_pre_p): - # lines are labels by 6 in this model - seperators_closeup=( (region_pre_p[:,:,:]==6))*1 - - seperators_closeup=seperators_closeup.astype(np.uint8) - imgray = cv2.cvtColor(seperators_closeup, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) + def do_prediction(self, patches, img, model, marginal_of_patch_percent=0.1): - contours_lines,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=self.find_features_of_lines(contours_lines) + img_height_model = model.layers[len(model.layers) - 1].output_shape[1] + img_width_model = model.layers[len(model.layers) - 1].output_shape[2] + n_classes = model.layers[len(model.layers) - 1].output_shape[3] - slope_lines_org_hor=slope_lines_org[slope_lines==0] - args=np.array( range(len(slope_lines) )) - len_x=seperators_closeup.shape[1]/4.0 + if patches: + if img.shape[0] < img_height_model: + img = resize_image(img, img_height_model, img.shape[1]) - args_hor=args[slope_lines==0] - dist_x_hor=dist_x[slope_lines==0] - x_min_main_hor=x_min_main[slope_lines==0] - x_max_main_hor=x_max_main[slope_lines==0] - cy_main_hor=cy_main[slope_lines==0] + if img.shape[1] < img_width_model: + img = resize_image(img, img.shape[0], img_width_model) - args_hor=args_hor[dist_x_hor>=len_x/2.0] - x_max_main_hor=x_max_main_hor[dist_x_hor>=len_x/2.0] - x_min_main_hor=x_min_main_hor[dist_x_hor>=len_x/2.0] - cy_main_hor=cy_main_hor[dist_x_hor>=len_x/2.0] - slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x/2.0] + # print(img_height_model,img_width_model) + # margin = int(0.2 * img_width_model) + margin = int(marginal_of_patch_percent * img_height_model) + width_mid = img_width_model - 2 * margin + height_mid = img_height_model - 2 * margin - slope_lines_org_hor=slope_lines_org_hor[np.abs(slope_lines_org_hor)<1.2] - slope_mean_hor=np.mean(slope_lines_org_hor) + img = img / float(255.0) + # print(sys.getsizeof(img)) + # print(np.max(img)) - if np.abs(slope_mean_hor)>1.2: - slope_mean_hor=0 + img = img.astype(np.float16) - #deskewed_new=rotate_image(image_regions_eraly_p[:,:,:],slope_mean_hor) + # print(sys.getsizeof(img)) - - args_ver=args[slope_lines==1] - y_min_main_ver=y_min_main[slope_lines==1] - y_max_main_ver=y_max_main[slope_lines==1] - x_min_main_ver=x_min_main[slope_lines==1] - x_max_main_ver=x_max_main[slope_lines==1] - cx_main_ver=cx_main[slope_lines==1] - dist_y_ver=y_max_main_ver-y_min_main_ver - len_y=seperators_closeup.shape[0]/3.0 - - return slope_mean_hor,cx_main_ver,dist_y_ver - - def return_contours_of_interested_region(self,region_pre_p,pixel,min_area=0.0002): - - # pixels of images are identified by 5 - if len(region_pre_p.shape)==3: - cnts_images=(region_pre_p[:,:,0]==pixel)*1 - else: - cnts_images=(region_pre_p[:,:]==pixel)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=1,min_area=min_area) - - return contours_imgs - - - def return_contours_of_interested_region_by_min_size(self,region_pre_p,pixel,min_size=0.00003): - - # pixels of images are identified by 5 - if len(region_pre_p.shape)==3: - cnts_images=(region_pre_p[:,:,0]==pixel)*1 - else: - cnts_images=(region_pre_p[:,:]==pixel)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=1,min_area=min_size) - - return contours_imgs - - - def return_contours_of_interested_textline(self,region_pre_p,pixel): - - # pixels of images are identified by 5 - if len(region_pre_p.shape)==3: - cnts_images=(region_pre_p[:,:,0]==pixel)*1 - else: - cnts_images=(region_pre_p[:,:]==pixel)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=1,min_area=0.000000003) - return contours_imgs - def find_images_contours_and_replace_table_and_graphic_pixels_by_image(self,region_pre_p): - - # pixels of images are identified by 5 - cnts_images=(region_pre_p[:,:,0]==5)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - #print(len(contours_imgs),'contours_imgs') - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=1,min_area=0.0003) - - #print(len(contours_imgs),'contours_imgs') - - boxes_imgs=self.return_bonding_box_of_contours(contours_imgs) - - for i in range(len(boxes_imgs)): - x1=int(boxes_imgs[i][0] ) - x2=int(boxes_imgs[i][0]+ boxes_imgs[i][2]) - y1=int(boxes_imgs[i][1] ) - y2=int(boxes_imgs[i][1]+ boxes_imgs[i][3]) - region_pre_p[y1:y2,x1:x2,0][region_pre_p[y1:y2,x1:x2,0]==8]=5 - region_pre_p[y1:y2,x1:x2,0][region_pre_p[y1:y2,x1:x2,0]==7]=5 - return region_pre_p - - def do_prediction(self,patches,img,model,marginal_of_patch_percent=0.1): - - img_height_model = model.layers[len(model.layers) - 1].output_shape[1] - img_width_model = model.layers[len(model.layers) - 1].output_shape[2] - n_classes = model.layers[len(model.layers) - 1].output_shape[3] - - - - if patches: - if img.shape[0] img_h: index_y_u = img_h index_y_d = img_h - img_height_model - - img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - label_p_pred = model.predict( - img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2])) + label_p_pred = model.predict(img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2])) seg = np.argmax(label_p_pred, axis=3)[0] seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) - if i==0 and j==0: - seg_color = seg_color[0:seg_color.shape[0] - margin, 0:seg_color.shape[1] - margin, :] - seg = seg[0:seg.shape[0] - margin, 0:seg.shape[1] - margin] - - mask_true[index_y_d + 0:index_y_u - margin, index_x_d + 0:index_x_u - margin] = seg - prediction_true[index_y_d + 0:index_y_u - margin, index_x_d + 0:index_x_u - margin, - :] = seg_color - - elif i==nxf-1 and j==nyf-1: - seg_color = seg_color[margin:seg_color.shape[0] - 0, margin:seg_color.shape[1] - 0, :] - seg = seg[margin:seg.shape[0] - 0, margin:seg.shape[1] - 0] - - mask_true[index_y_d + margin:index_y_u - 0, index_x_d + margin:index_x_u - 0] = seg - prediction_true[index_y_d + margin:index_y_u - 0, index_x_d + margin:index_x_u - 0, - :] = seg_color - - elif i==0 and j==nyf-1: - seg_color = seg_color[margin:seg_color.shape[0] - 0, 0:seg_color.shape[1] - margin, :] - seg = seg[margin:seg.shape[0] - 0, 0:seg.shape[1] - margin] - - mask_true[index_y_d + margin:index_y_u - 0, index_x_d + 0:index_x_u - margin] = seg - prediction_true[index_y_d + margin:index_y_u - 0, index_x_d + 0:index_x_u - margin, - :] = seg_color - - elif i==nxf-1 and j==0: - seg_color = seg_color[0:seg_color.shape[0] - margin, margin:seg_color.shape[1] - 0, :] - seg = seg[0:seg.shape[0] - margin, margin:seg.shape[1] - 0] - - mask_true[index_y_d + 0:index_y_u - margin, index_x_d + margin:index_x_u - 0] = seg - prediction_true[index_y_d + 0:index_y_u - margin, index_x_d + margin:index_x_u - 0, - :] = seg_color - - elif i==0 and j!=0 and j!=nyf-1: - seg_color = seg_color[margin:seg_color.shape[0] - margin, 0:seg_color.shape[1] - margin, :] - seg = seg[margin:seg.shape[0] - margin, 0:seg.shape[1] - margin] - - mask_true[index_y_d + margin:index_y_u - margin, index_x_d + 0:index_x_u - margin] = seg - prediction_true[index_y_d + margin:index_y_u - margin, index_x_d + 0:index_x_u - margin, - :] = seg_color - - elif i==nxf-1 and j!=0 and j!=nyf-1: - seg_color = seg_color[margin:seg_color.shape[0] - margin, margin:seg_color.shape[1] - 0, :] - seg = seg[margin:seg.shape[0] - margin, margin:seg.shape[1] - 0] - - mask_true[index_y_d + margin:index_y_u - margin, index_x_d + margin:index_x_u - 0] = seg - prediction_true[index_y_d + margin:index_y_u - margin, index_x_d + margin:index_x_u - 0, - :] = seg_color - - elif i!=0 and i!=nxf-1 and j==0: - seg_color = seg_color[0:seg_color.shape[0] - margin, margin:seg_color.shape[1] - margin, :] - seg = seg[0:seg.shape[0] - margin, margin:seg.shape[1] - margin] - - mask_true[index_y_d + 0:index_y_u - margin, index_x_d + margin:index_x_u - margin] = seg - prediction_true[index_y_d + 0:index_y_u - margin, index_x_d + margin:index_x_u - margin, - :] = seg_color - - elif i!=0 and i!=nxf-1 and j==nyf-1: - seg_color = seg_color[margin:seg_color.shape[0] - 0, margin:seg_color.shape[1] - margin, :] - seg = seg[margin:seg.shape[0] - 0, margin:seg.shape[1] - margin] - - mask_true[index_y_d + margin:index_y_u - 0, index_x_d + margin:index_x_u - margin] = seg - prediction_true[index_y_d + margin:index_y_u - 0, index_x_d + margin:index_x_u - margin, - :] = seg_color + if i == 0 and j == 0: + seg_color = seg_color[0 : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] + seg = seg[0 : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + + mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin] = seg + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg_color + + elif i == nxf - 1 and j == nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - 0, :] + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - 0] + + mask_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0] = seg + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0, :] = seg_color + + elif i == 0 and j == nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - 0, 0 : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - 0, 0 : seg.shape[1] - margin] + + mask_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin, :] = seg_color + + elif i == nxf - 1 and j == 0: + seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - 0] + + mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0] = seg + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg_color + + elif i == 0 and j != 0 and j != nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + + mask_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg_color + + elif i == nxf - 1 and j != 0 and j != nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - 0] + + mask_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg_color + + elif i != 0 and i != nxf - 1 and j == 0: + seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - margin] + + mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin] = seg + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg_color + + elif i != 0 and i != nxf - 1 and j == nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - margin] + + mask_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin, :] = seg_color else: - seg_color = seg_color[margin:seg_color.shape[0] - margin, margin:seg_color.shape[1] - margin, :] - seg = seg[margin:seg.shape[0] - margin, margin:seg.shape[1] - margin] + seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - margin] - mask_true[index_y_d + margin:index_y_u - margin, index_x_d + margin:index_x_u - margin] = seg - prediction_true[index_y_d + margin:index_y_u - margin, index_x_d + margin:index_x_u - margin, - :] = seg_color + mask_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg_color prediction_true = prediction_true.astype(np.uint8) del img @@ -1171,43 +760,40 @@ class eynollah: del seg_color del seg del img_patch - + if not patches: - img_h_page=img.shape[0] - img_w_page=img.shape[1] - img = img /float( 255.0) - img = self.resize_image(img, img_height_model, img_width_model) + img_h_page = img.shape[0] + img_w_page = img.shape[1] + img = img / float(255.0) + img = resize_image(img, img_height_model, img_width_model) - label_p_pred = model.predict( - img.reshape(1, img.shape[0], img.shape[1], img.shape[2])) + label_p_pred = model.predict(img.reshape(1, img.shape[0], img.shape[1], img.shape[2])) seg = np.argmax(label_p_pred, axis=3)[0] - seg_color =np.repeat(seg[:, :, np.newaxis], 3, axis=2) - prediction_true = self.resize_image(seg_color, img_h_page, img_w_page) + seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) + prediction_true = resize_image(seg_color, img_h_page, img_w_page) prediction_true = prediction_true.astype(np.uint8) - - + del img del seg_color del label_p_pred del seg del model gc.collect() - + return prediction_true - + def early_page_for_num_of_column_classification(self): - img=cv2.imread(self.image_dir) - img=img.astype(np.uint8) - patches=False + img = cv2.imread(self.image_dir) + img = img.astype(np.uint8) + patches = False model_page, session_page = self.start_new_session_and_model(self.model_page_dir) - ###img = self.otsu_copy(self.image) + ###img = otsu_copy(self.image) for ii in range(1): img = cv2.GaussianBlur(img, (5, 5), 0) - - img_page_prediction=self.do_prediction(patches,img,model_page) - + img_page_prediction = self.do_prediction(patches, img, model_page) + imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) _, thresh = cv2.threshold(imgray, 0, 255, 0) @@ -1222,7 +808,7 @@ class eynollah: box = [x, y, w, h] - croped_page, page_coord = self.crop_image_inside_box(box, img) + croped_page, page_coord = crop_image_inside_box(box, img) session_page.close() del model_page @@ -1239,21 +825,19 @@ class eynollah: del h del imgray del img_page_prediction - gc.collect() return croped_page, page_coord def extract_page(self): - patches=False + patches = False model_page, session_page = self.start_new_session_and_model(self.model_page_dir) - ###img = self.otsu_copy(self.image) + ###img = otsu_copy(self.image) for ii in range(1): img = cv2.GaussianBlur(self.image, (5, 5), 0) - - img_page_prediction=self.do_prediction(patches,img,model_page) - + img_page_prediction = self.do_prediction(patches, img, model_page) + imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) _, thresh = cv2.threshold(imgray, 0, 255, 0) @@ -1265,31 +849,25 @@ class eynollah: cnt = contours[np.argmax(cnt_size)] x, y, w, h = cv2.boundingRect(cnt) - - if x<=30: - w=w+x - x=0 - if (self.image.shape[1]-(x+w) )<=30: - w=w+(self.image.shape[1]-(x+w) ) - - if y<=30: - h=h+y - y=0 - if (self.image.shape[0]-(y+h) )<=30: - h=h+(self.image.shape[0]-(y+h) ) - - + if x <= 30: + w = w + x + x = 0 + if (self.image.shape[1] - (x + w)) <= 30: + w = w + (self.image.shape[1] - (x + w)) + + if y <= 30: + h = h + y + y = 0 + if (self.image.shape[0] - (y + h)) <= 30: + h = h + (self.image.shape[0] - (y + h)) box = [x, y, w, h] - croped_page, page_coord = self.crop_image_inside_box(box, self.image) - - self.cont_page=[] - self.cont_page.append( np.array( [ [ page_coord[2] , page_coord[0] ] , - [ page_coord[3] , page_coord[0] ] , - [ page_coord[3] , page_coord[1] ] , - [ page_coord[2] , page_coord[1] ]] ) ) + croped_page, page_coord = crop_image_inside_box(box, self.image) + + self.cont_page = [] + self.cont_page.append(np.array([[page_coord[2], page_coord[0]], [page_coord[3], page_coord[0]], [page_coord[3], page_coord[1]], [page_coord[2], page_coord[1]]])) session_page.close() del model_page @@ -1301,316 +879,154 @@ class eynollah: gc.collect() return croped_page, page_coord - - def extract_drop_capital_13(self, img,patches,cols): - - - img_height_h=img.shape[0] - img_width_h=img.shape[1] - patches=False - - - - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully_np) - - img_1=img[:int(img.shape[0]/3.),:,:] - img_2=img[int(img.shape[0]/3.):int(2*img.shape[0]/3.),:,:] - img_3=img[int(2*img.shape[0]/3.):,:,:] - - - - #img_1 = self.otsu_copy_binary(img_1)#self.otsu_copy(img) - #img_1 = img_1.astype(np.uint16) - - plt.imshow(img_1) - plt.show() - #img_2 = self.otsu_copy_binary(img_2)#self.otsu_copy(img) - #img_2 = img_2.astype(np.uint16) - - plt.imshow(img_2) - plt.show() - #img_3 = self.otsu_copy_binary(img_3)#self.otsu_copy(img) - #img_3 = img_3.astype(np.uint16) - - - plt.imshow(img_3) - plt.show() - - prediction_regions_1=self.do_prediction(patches,img_1,model_region) - - - plt.imshow(prediction_regions_1) - plt.show() - - prediction_regions_2=self.do_prediction(patches,img_2,model_region) - - plt.imshow(prediction_regions_2) - plt.show() - prediction_regions_3=self.do_prediction(patches,img_3,model_region) - - - plt.imshow(prediction_regions_3) - plt.show() - prediction_regions=np.zeros((img_height_h,img_width_h)) - - prediction_regions[:int(img.shape[0]/3.),:]=prediction_regions_1[:,:,0] - prediction_regions[int(img.shape[0]/3.):int(2*img.shape[0]/3.),:]=prediction_regions_2[:,:,0] - prediction_regions[int(2*img.shape[0]/3.):,:]=prediction_regions_3[:,:,0] - - - - session_region.close() - del img_1 - del img_2 - del img_3 - del prediction_regions_1 - del prediction_regions_2 - del prediction_regions_3 - del model_region - del session_region - del img - gc.collect() - return prediction_regions - - - def extract_text_regions(self, img,patches,cols): - img_height_h=img.shape[0] - img_width_h=img.shape[1] + def extract_text_regions(self, img, patches, cols): + img_height_h = img.shape[0] + img_width_h = img.shape[1] ###if patches and cols>=3 : - ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully) + ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully) ###if not patches: - ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully_np) - + ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully_np) + ###if patches and cols==2 : - ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_2col) - + ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_2col) + ###if patches and cols==1 : - ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_2col) - + ###model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_2col) - ###if patches and cols>=2: - - ###img = self.otsu_copy_binary(img)#self.otsu_copy(img) - ###img = img.astype(np.uint8) - - + + ###img = otsu_copy_binary(img)#otsu_copy(img) + ###img = img.astype(np.uint8) + ###if patches and cols==1: - - ###img = self.otsu_copy_binary(img)#self.otsu_copy(img) - ###img = img.astype(np.uint8) - ###img= self.resize_image(img, int(img_height_h*1), int(img_width_h*1) ) - - - if patches : + + ###img = otsu_copy_binary(img)#otsu_copy(img) + ###img = img.astype(np.uint8) + ###img= resize_image(img, int(img_height_h*1), int(img_width_h*1) ) + + if patches: model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully) if not patches: model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully_np) - - if patches and cols==1: - img2 = self.otsu_copy_binary(img)#self.otsu_copy(img) + + if patches and cols == 1: + img2 = otsu_copy_binary(img) # otsu_copy(img) img2 = img2.astype(np.uint8) - img2= self.resize_image(img2, int(img_height_h*0.7), int(img_width_h*0.7) ) - - marginal_of_patch_percent=0.1 - prediction_regions2=self.do_prediction(patches,img2,model_region,marginal_of_patch_percent) - prediction_regions2=self.resize_image(prediction_regions2, img_height_h, img_width_h ) - - if patches and cols==2: - img2 = self.otsu_copy_binary(img)#self.otsu_copy(img) + img2 = resize_image(img2, int(img_height_h * 0.7), int(img_width_h * 0.7)) + + marginal_of_patch_percent = 0.1 + prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent) + prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) + + if patches and cols == 2: + img2 = otsu_copy_binary(img) # otsu_copy(img) img2 = img2.astype(np.uint8) - img2= self.resize_image(img2, int(img_height_h*0.4), int(img_width_h*0.4) ) - - marginal_of_patch_percent=0.1 - prediction_regions2=self.do_prediction(patches,img2,model_region,marginal_of_patch_percent) - prediction_regions2=self.resize_image(prediction_regions2, img_height_h, img_width_h ) - elif patches and cols>2: - img2 = self.otsu_copy_binary(img)#self.otsu_copy(img) + img2 = resize_image(img2, int(img_height_h * 0.4), int(img_width_h * 0.4)) + + marginal_of_patch_percent = 0.1 + prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent) + prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) + elif patches and cols > 2: + img2 = otsu_copy_binary(img) # otsu_copy(img) img2 = img2.astype(np.uint8) - img2= self.resize_image(img2, int(img_height_h*0.3), int(img_width_h*0.3) ) - - marginal_of_patch_percent=0.1 - prediction_regions2=self.do_prediction(patches,img2,model_region,marginal_of_patch_percent) - prediction_regions2=self.resize_image(prediction_regions2, img_height_h, img_width_h ) + img2 = resize_image(img2, int(img_height_h * 0.3), int(img_width_h * 0.3)) + + marginal_of_patch_percent = 0.1 + prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent) + prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) + + if patches and cols == 2: + img = otsu_copy_binary(img) # otsu_copy(img) - - if patches and cols==2: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint8) - - if img_width_h>=2000: - img= self.resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) + + if img_width_h >= 2000: + img = resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)) else: - pass#img= self.resize_image(img, int(img_height_h*1), int(img_width_h*1) ) + pass # img= resize_image(img, int(img_height_h*1), int(img_width_h*1) ) img = img.astype(np.uint8) - - if patches and cols==1: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - + + if patches and cols == 1: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint8) - img= self.resize_image(img, int(img_height_h*0.5), int(img_width_h*0.5) ) + img = resize_image(img, int(img_height_h * 0.5), int(img_width_h * 0.5)) img = img.astype(np.uint8) - - if patches and cols==3: - - img = self.otsu_copy_binary(img)#self.otsu_copy(img) + + if patches and cols == 3: + + img = otsu_copy_binary(img) # otsu_copy(img) img = img.astype(np.uint8) - #img= self.resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) - - if patches and cols==4: - - img = self.otsu_copy_binary(img)#self.otsu_copy(img) + # img= resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) + + if patches and cols == 4: + + img = otsu_copy_binary(img) # otsu_copy(img) img = img.astype(np.uint8) - #img= self.resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) - - if patches and cols>=5: - - img = self.otsu_copy_binary(img)#self.otsu_copy(img) + # img= resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) + + if patches and cols >= 5: + + img = otsu_copy_binary(img) # otsu_copy(img) img = img.astype(np.uint8) - #img= self.resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) + # img= resize_image(img, int(img_height_h*0.9), int(img_width_h*0.9) ) if not patches: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) + img = otsu_copy_binary(img) # otsu_copy(img) img = img.astype(np.uint8) - prediction_regions2=None - - marginal_of_patch_percent=0.1 - prediction_regions=self.do_prediction(patches,img,model_region,marginal_of_patch_percent) - prediction_regions=self.resize_image(prediction_regions, img_height_h, img_width_h ) - + prediction_regions2 = None + + marginal_of_patch_percent = 0.1 + prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent) + prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) - - session_region.close() del model_region del session_region del img gc.collect() - return prediction_regions,prediction_regions2 - - def extract_only_text_regions(self, img,patches): - - model_region, session_region = self.start_new_session_and_model(self.model_only_text) - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint8) - img_org=np.copy(img) - - img_h=img_org.shape[0] - img_w=img_org.shape[1] - - img= self.resize_image(img_org, int(img_org.shape[0]*1), int(img_org.shape[1]*1)) - + return prediction_regions, prediction_regions2 - prediction_regions1=self.do_prediction(patches,img,model_region) - - prediction_regions1= self.resize_image(prediction_regions1, img_h, img_w) - - - #prediction_regions1 = cv2.dilate(prediction_regions1, self.kernel, iterations=4) - #prediction_regions1 = cv2.erode(prediction_regions1, self.kernel, iterations=7) - #prediction_regions1 = cv2.dilate(prediction_regions1, self.kernel, iterations=2) - - - img= self.resize_image(img_org, int(img_org.shape[0]*1), int(img_org.shape[1]*1)) - - - prediction_regions2=self.do_prediction(patches,img,model_region) - - prediction_regions2= self.resize_image(prediction_regions2, img_h, img_w) - - - #prediction_regions2 = cv2.dilate(prediction_regions2, self.kernel, iterations=2) - prediction_regions2 = cv2.erode(prediction_regions2, self.kernel, iterations=2) - prediction_regions2 = cv2.dilate(prediction_regions2, self.kernel, iterations=2) - - - #prediction_regions=( (prediction_regions2[:,:,0]==1) & (prediction_regions1[:,:,0]==1) ) - #prediction_regions=(prediction_regions1[:,:,0]==1) - - session_region.close() - del model_region - del session_region - gc.collect() - return prediction_regions1[:,:,0] - - def extract_binarization(self, img,patches): - - model_bin, session_bin = self.start_new_session_and_model(self.model_binafrization) + def get_slopes_and_deskew_new(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): + num_cores = cpu_count() + queue_of_all_params = Queue() - - img_h=img.shape[0] - img_w=img.shape[1] - - img= self.resize_image(img, int(img.shape[0]*1), int(img.shape[1]*1)) - + processes = [] + nh = np.linspace(0, len(boxes), num_cores + 1) + indexes_by_text_con = np.array(range(len(contours_par))) - prediction_regions=self.do_prediction(patches,img,model_bin) - - res=(prediction_regions[:,:,0]!=0)*1 + for i in range(num_cores): + boxes_per_process = boxes[int(nh[i]) : int(nh[i + 1])] + contours_per_process = contours[int(nh[i]) : int(nh[i + 1])] + contours_par_per_process = contours_par[int(nh[i]) : int(nh[i + 1])] + indexes_text_con_per_process = indexes_by_text_con[int(nh[i]) : int(nh[i + 1])] - + processes.append(Process(target=self.do_work_of_slopes_new, args=(queue_of_all_params, boxes_per_process, textline_mask_tot, contours_per_process, contours_par_per_process, indexes_text_con_per_process, image_page_rotated, slope_deskew))) - - img_fin=np.zeros((res.shape[0],res.shape[1],3) ) - res[:,:][res[:,:]==0]=2 - res=res-1 - res=res*255 - img_fin[:,:,0]=res - img_fin[:,:,1]=res - img_fin[:,:,2]=res - - session_bin.close() - del model_bin - del session_bin - gc.collect() - #plt.imshow(img_fin[:,:,0]) - #plt.show() - return img_fin - - def get_slopes_and_deskew_new(self, contours,contours_par,textline_mask_tot,image_page_rotated,boxes,slope_deskew): - num_cores =cpu_count() - queue_of_all_params = Queue() - - processes = [] - nh=np.linspace(0, len(boxes), num_cores+1) - indexes_by_text_con=np.array(range(len(contours_par))) - - for i in range(num_cores): - boxes_per_process=boxes[int(nh[i]):int(nh[i+1])] - contours_per_process=contours[int(nh[i]):int(nh[i+1])] - contours_par_per_process=contours_par[int(nh[i]):int(nh[i+1])] - indexes_text_con_per_process=indexes_by_text_con[int(nh[i]):int(nh[i+1])] - - processes.append(Process(target=self.do_work_of_slopes_new, args=(queue_of_all_params, boxes_per_process, textline_mask_tot, contours_per_process,contours_par_per_process,indexes_text_con_per_process,image_page_rotated,slope_deskew))) - for i in range(num_cores): processes[i].start() - + slopes = [] - all_found_texline_polygons=[] - all_found_text_regions=[] - all_found_text_regions_par=[] - boxes=[] - all_box_coord=[] - all_index_text_con=[] - + all_found_texline_polygons = [] + all_found_text_regions = [] + all_found_text_regions_par = [] + boxes = [] + all_box_coord = [] + all_index_text_con = [] + for i in range(num_cores): - list_all_par=queue_of_all_params.get(True) - - slopes_for_sub_process=list_all_par[0] - polys_for_sub_process=list_all_par[1] - boxes_for_sub_process=list_all_par[2] - contours_for_subprocess=list_all_par[3] - contours_par_for_subprocess=list_all_par[4] - boxes_coord_for_subprocess=list_all_par[5] - indexes_for_subprocess=list_all_par[6] - + list_all_par = queue_of_all_params.get(True) + + slopes_for_sub_process = list_all_par[0] + polys_for_sub_process = list_all_par[1] + boxes_for_sub_process = list_all_par[2] + contours_for_subprocess = list_all_par[3] + contours_par_for_subprocess = list_all_par[4] + boxes_coord_for_subprocess = list_all_par[5] + indexes_for_subprocess = list_all_par[6] + for j in range(len(slopes_for_sub_process)): slopes.append(slopes_for_sub_process[j]) all_found_texline_polygons.append(polys_for_sub_process[j]) @@ -1619,52 +1035,50 @@ class eynollah: all_found_text_regions_par.append(contours_par_for_subprocess[j]) all_box_coord.append(boxes_coord_for_subprocess[j]) all_index_text_con.append(indexes_for_subprocess[j]) - + for i in range(num_cores): processes[i].join() - #print(slopes,'slopes') - return slopes,all_found_texline_polygons,boxes,all_found_text_regions,all_found_text_regions_par,all_box_coord,all_index_text_con - - - def get_slopes_and_deskew_new_curved(self, contours,contours_par,textline_mask_tot,image_page_rotated,boxes,mask_texts_only,num_col,scale_par,slope_deskew): - num_cores =cpu_count() + # print(slopes,'slopes') + return slopes, all_found_texline_polygons, boxes, all_found_text_regions, all_found_text_regions_par, all_box_coord, all_index_text_con + + def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, mask_texts_only, num_col, scale_par, slope_deskew): + num_cores = cpu_count() queue_of_all_params = Queue() - + processes = [] - nh=np.linspace(0, len(boxes), num_cores+1) - indexes_by_text_con=np.array(range(len(contours_par))) - - + nh = np.linspace(0, len(boxes), num_cores + 1) + indexes_by_text_con = np.array(range(len(contours_par))) + for i in range(num_cores): - boxes_per_process=boxes[int(nh[i]):int(nh[i+1])] - contours_per_process=contours[int(nh[i]):int(nh[i+1])] - contours_par_per_process=contours_par[int(nh[i]):int(nh[i+1])] - indexes_text_con_per_process=indexes_by_text_con[int(nh[i]):int(nh[i+1])] - - processes.append(Process(target=self.do_work_of_slopes_new_curved, args=(queue_of_all_params, boxes_per_process, textline_mask_tot, contours_per_process,contours_par_per_process,image_page_rotated,mask_texts_only,num_col,scale_par,indexes_text_con_per_process,slope_deskew))) - + boxes_per_process = boxes[int(nh[i]) : int(nh[i + 1])] + contours_per_process = contours[int(nh[i]) : int(nh[i + 1])] + contours_par_per_process = contours_par[int(nh[i]) : int(nh[i + 1])] + indexes_text_con_per_process = indexes_by_text_con[int(nh[i]) : int(nh[i + 1])] + + processes.append(Process(target=self.do_work_of_slopes_new_curved, args=(queue_of_all_params, boxes_per_process, textline_mask_tot, contours_per_process, contours_par_per_process, image_page_rotated, mask_texts_only, num_col, scale_par, indexes_text_con_per_process, slope_deskew))) + for i in range(num_cores): processes[i].start() - + slopes = [] - all_found_texline_polygons=[] - all_found_text_regions=[] - all_found_text_regions_par=[] - boxes=[] - all_box_coord=[] - all_index_text_con=[] - + all_found_texline_polygons = [] + all_found_text_regions = [] + all_found_text_regions_par = [] + boxes = [] + all_box_coord = [] + all_index_text_con = [] + for i in range(num_cores): - list_all_par=queue_of_all_params.get(True) - - polys_for_sub_process=list_all_par[0] - boxes_for_sub_process=list_all_par[1] - contours_for_subprocess=list_all_par[2] - contours_par_for_subprocess=list_all_par[3] - boxes_coord_for_subprocess=list_all_par[4] - indexes_for_subprocess=list_all_par[5] - slopes_for_sub_process=list_all_par[6] - + list_all_par = queue_of_all_params.get(True) + + polys_for_sub_process = list_all_par[0] + boxes_for_sub_process = list_all_par[1] + contours_for_subprocess = list_all_par[2] + contours_par_for_subprocess = list_all_par[3] + boxes_coord_for_subprocess = list_all_par[4] + indexes_for_subprocess = list_all_par[5] + slopes_for_sub_process = list_all_par[6] + for j in range(len(polys_for_sub_process)): slopes.append(slopes_for_sub_process[j]) all_found_texline_polygons.append(polys_for_sub_process[j]) @@ -1673,196 +1087,173 @@ class eynollah: all_found_text_regions_par.append(contours_par_for_subprocess[j]) all_box_coord.append(boxes_coord_for_subprocess[j]) all_index_text_con.append(indexes_for_subprocess[j]) - + for i in range(num_cores): processes[i].join() - #print(slopes,'slopes') - return all_found_texline_polygons,boxes,all_found_text_regions,all_found_text_regions_par,all_box_coord,all_index_text_con,slopes - - - - def do_work_of_slopes_new_curved(self,queue_of_all_params,boxes_text,textline_mask_tot_ea,contours_per_process,contours_par_per_process,image_page_rotated,mask_texts_only,num_col,scale_par,indexes_r_con_per_pro,slope_deskew): + # print(slopes,'slopes') + return all_found_texline_polygons, boxes, all_found_text_regions, all_found_text_regions_par, all_box_coord, all_index_text_con, slopes + + def do_work_of_slopes_new_curved(self, queue_of_all_params, boxes_text, textline_mask_tot_ea, contours_per_process, contours_par_per_process, image_page_rotated, mask_texts_only, num_col, scale_par, indexes_r_con_per_pro, slope_deskew): slopes_per_each_subprocess = [] - bounding_box_of_textregion_per_each_subprocess=[] - textlines_rectangles_per_each_subprocess=[] - contours_textregion_per_each_subprocess=[] - contours_textregion_par_per_each_subprocess=[] - all_box_coord_per_process=[] - index_by_text_region_contours=[] - slope_biggest=0 - - textline_cnt_seperated=np.zeros(textline_mask_tot_ea.shape) - - + bounding_box_of_textregion_per_each_subprocess = [] + textlines_rectangles_per_each_subprocess = [] + contours_textregion_per_each_subprocess = [] + contours_textregion_par_per_each_subprocess = [] + all_box_coord_per_process = [] + index_by_text_region_contours = [] + slope_biggest = 0 + + textline_cnt_seperated = np.zeros(textline_mask_tot_ea.shape) + for mv in range(len(boxes_text)): - - all_text_region_raw=textline_mask_tot_ea[boxes_text[mv][1]:boxes_text[mv][1]+boxes_text[mv][3] , boxes_text[mv][0]:boxes_text[mv][0]+boxes_text[mv][2] ] - all_text_region_raw=all_text_region_raw.astype(np.uint8) - - img_int_p=all_text_region_raw[:,:]#self.all_text_region_raw[mv] - + + all_text_region_raw = textline_mask_tot_ea[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]] + all_text_region_raw = all_text_region_raw.astype(np.uint8) + + img_int_p = all_text_region_raw[:, :] # self.all_text_region_raw[mv] + ##img_int_p=cv2.erode(img_int_p,self.kernel,iterations = 2) - - #plt.imshow(img_int_p) - #plt.show() - - if img_int_p.shape[0]/img_int_p.shape[1]<0.1: - + + # plt.imshow(img_int_p) + # plt.show() + + if img_int_p.shape[0] / img_int_p.shape[1] < 0.1: + slopes_per_each_subprocess.append(0) - - slope_first=0 - slope_for_all=[slope_deskew][0] - + + slope_first = 0 + slope_for_all = [slope_deskew][0] + else: - - + try: - textline_con,hierachy=self.return_contours_of_image(img_int_p) - textline_con_fil=self.filter_contours_area_of_image(img_int_p,textline_con,hierachy,max_area=1,min_area=0.0008) - y_diff_mean=self.find_contours_mean_y_diff(textline_con_fil) - - sigma_des=int( y_diff_mean * (4./40.0) ) - - if sigma_des<1: - sigma_des=1 - - img_int_p[img_int_p>0]=1 - #slope_for_all=self.return_deskew_slope_new(img_int_p,sigma_des) - slope_for_all=self.return_deskew_slop(img_int_p,sigma_des) - - if abs(slope_for_all)<0.5: - slope_for_all=[slope_deskew][0] - #old method - #slope_for_all=self.textline_contours_to_get_slope_correctly(self.all_text_region_raw[mv],denoised,contours[mv]) - #text_patch_processed=textline_contours_postprocessing(gada) - + textline_con, hierachy = return_contours_of_image(img_int_p) + textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, hierachy, max_area=1, min_area=0.0008) + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) + + sigma_des = int(y_diff_mean * (4.0 / 40.0)) + + if sigma_des < 1: + sigma_des = 1 + + img_int_p[img_int_p > 0] = 1 + # slope_for_all=self.return_deskew_slope_new(img_int_p,sigma_des) + slope_for_all = return_deskew_slop(img_int_p, sigma_des, dir_of_all=self.dir_of_all, f_name=self.f_name) + + if abs(slope_for_all) < 0.5: + slope_for_all = [slope_deskew][0] + # old method + # slope_for_all=self.textline_contours_to_get_slope_correctly(self.all_text_region_raw[mv],denoised,contours[mv]) + # text_patch_processed=textline_contours_postprocessing(gada) + except: - slope_for_all=999 - - - ##slope_for_all=self.return_deskew_slop(img_int_p,sigma_des) - - if slope_for_all==999: - slope_for_all=[slope_deskew][0] + slope_for_all = 999 + + ##slope_for_all=return_deskew_slop(img_int_p,sigma_des, dir_of_all=self.dir_of_all, f_name=self.f_name) + + if slope_for_all == 999: + slope_for_all = [slope_deskew][0] ##if np.abs(slope_for_all)>32.5 and slope_for_all!=999: - ##slope_for_all=slope_biggest + ##slope_for_all=slope_biggest ##elif slope_for_all==999: - ##slope_for_all=slope_biggest + ##slope_for_all=slope_biggest slopes_per_each_subprocess.append(slope_for_all) - - - + index_by_text_region_contours.append(indexes_r_con_per_pro[mv]) - - crop_img,crop_coor=self.crop_image_inside_box(boxes_text[mv],image_page_rotated) - if abs(slope_for_all)<45: + crop_img, crop_coor = crop_image_inside_box(boxes_text[mv], image_page_rotated) + if abs(slope_for_all) < 45: + + # all_box_coord.append(crop_coor) + + textline_region_in_image = np.zeros(textline_mask_tot_ea.shape) + cnt_o_t_max = contours_par_per_process[mv] - #all_box_coord.append(crop_coor) - - textline_region_in_image=np.zeros(textline_mask_tot_ea.shape) - cnt_o_t_max=contours_par_per_process[mv] - x, y, w, h = cv2.boundingRect(cnt_o_t_max) - - - mask_biggest=np.zeros(mask_texts_only.shape) - mask_biggest=cv2.fillPoly(mask_biggest, pts =[cnt_o_t_max], color=(1,1,1)) - - mask_region_in_patch_region=mask_biggest[y:y+h,x:x+w] - - - textline_biggest_region=mask_biggest*textline_mask_tot_ea - - #print(slope_for_all,'slope_for_all') - textline_rotated_seperated=self.seperate_lines_new2(textline_biggest_region[y:y+h,x:x+w],0,num_col,slope_for_all) - - - #new line added + + mask_biggest = np.zeros(mask_texts_only.shape) + mask_biggest = cv2.fillPoly(mask_biggest, pts=[cnt_o_t_max], color=(1, 1, 1)) + + mask_region_in_patch_region = mask_biggest[y : y + h, x : x + w] + + textline_biggest_region = mask_biggest * textline_mask_tot_ea + + # print(slope_for_all,'slope_for_all') + textline_rotated_seperated = seperate_lines_new2(textline_biggest_region[y : y + h, x : x + w], 0, num_col, slope_for_all, self.dir_of_all, self.f_name) + + # new line added ##print(np.shape(textline_rotated_seperated),np.shape(mask_biggest)) - textline_rotated_seperated[mask_region_in_patch_region[:,:]!=1]=0 - #till here - - textline_cnt_seperated[y:y+h,x:x+w]=textline_rotated_seperated - textline_region_in_image[y:y+h,x:x+w]=textline_rotated_seperated - - #plt.imshow(textline_region_in_image) - #plt.show() - - #plt.imshow(textline_cnt_seperated) - #plt.show() - + textline_rotated_seperated[mask_region_in_patch_region[:, :] != 1] = 0 + # till here - - - pixel_img=1 - cnt_textlines_in_image=self.return_contours_of_interested_textline(textline_region_in_image,pixel_img) - - - - textlines_cnt_per_region=[] + textline_cnt_seperated[y : y + h, x : x + w] = textline_rotated_seperated + textline_region_in_image[y : y + h, x : x + w] = textline_rotated_seperated + + # plt.imshow(textline_region_in_image) + # plt.show() + + # plt.imshow(textline_cnt_seperated) + # plt.show() + + pixel_img = 1 + cnt_textlines_in_image = return_contours_of_interested_textline(textline_region_in_image, pixel_img) + + textlines_cnt_per_region = [] for jjjj in range(len(cnt_textlines_in_image)): - mask_biggest2=np.zeros(mask_texts_only.shape) - mask_biggest2=cv2.fillPoly(mask_biggest2, pts =[cnt_textlines_in_image[jjjj]], color=(1,1,1)) - if num_col+1==1: - mask_biggest2=cv2.dilate(mask_biggest2, self.kernel, iterations=5) + mask_biggest2 = np.zeros(mask_texts_only.shape) + mask_biggest2 = cv2.fillPoly(mask_biggest2, pts=[cnt_textlines_in_image[jjjj]], color=(1, 1, 1)) + if num_col + 1 == 1: + mask_biggest2 = cv2.dilate(mask_biggest2, self.kernel, iterations=5) else: - - mask_biggest2=cv2.dilate(mask_biggest2, self.kernel, iterations=4) - - pixel_img=1 - - mask_biggest2=self.resize_image(mask_biggest2,int(mask_biggest2.shape[0]*scale_par) ,int(mask_biggest2.shape[1]*scale_par) ) - - cnt_textlines_in_image_ind=self.return_contours_of_interested_textline(mask_biggest2,pixel_img) - + + mask_biggest2 = cv2.dilate(mask_biggest2, self.kernel, iterations=4) + + pixel_img = 1 + + mask_biggest2 = resize_image(mask_biggest2, int(mask_biggest2.shape[0] * scale_par), int(mask_biggest2.shape[1] * scale_par)) + + cnt_textlines_in_image_ind = return_contours_of_interested_textline(mask_biggest2, pixel_img) + try: - #textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]/scale_par) + # textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]/scale_par) textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]) except: pass - + else: - slope_first=0 - add_boxes_coor_into_textlines=True - textlines_cnt_per_region=self.textline_contours_postprocessing(all_text_region_raw,slope_for_all,contours_par_per_process[mv],boxes_text[mv],slope_first,add_boxes_coor_into_textlines) - add_boxes_coor_into_textlines=False - #print(np.shape(textlines_cnt_per_region),'textlines_cnt_per_region') - - - + slope_first = 0 + add_boxes_coor_into_textlines = True + textlines_cnt_per_region = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contours_par_per_process[mv], boxes_text[mv], slope_first, add_boxes_coor_into_textlines) + add_boxes_coor_into_textlines = False + # print(np.shape(textlines_cnt_per_region),'textlines_cnt_per_region') + + # textlines_cnt_tot_per_process.append(textlines_cnt_per_region) + # index_polygons_per_process_per_process.append(index_polygons_per_process[iiii]) - #textlines_cnt_tot_per_process.append(textlines_cnt_per_region) - #index_polygons_per_process_per_process.append(index_polygons_per_process[iiii]) - - textlines_rectangles_per_each_subprocess.append(textlines_cnt_per_region) - #all_found_texline_polygons.append(cnt_clean_rot) - bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv] ) - - - + # all_found_texline_polygons.append(cnt_clean_rot) + bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv]) + contours_textregion_per_each_subprocess.append(contours_per_process[mv]) contours_textregion_par_per_each_subprocess.append(contours_par_per_process[mv]) all_box_coord_per_process.append(crop_coor) - queue_of_all_params.put([textlines_rectangles_per_each_subprocess, bounding_box_of_textregion_per_each_subprocess, contours_textregion_per_each_subprocess,contours_textregion_par_per_each_subprocess,all_box_coord_per_process,index_by_text_region_contours,slopes_per_each_subprocess]) - def do_work_of_slopes_new(self,queue_of_all_params,boxes_text,textline_mask_tot_ea,contours_per_process,contours_par_per_process,indexes_r_con_per_pro,image_page_rotated,slope_deskew): - + queue_of_all_params.put([textlines_rectangles_per_each_subprocess, bounding_box_of_textregion_per_each_subprocess, contours_textregion_per_each_subprocess, contours_textregion_par_per_each_subprocess, all_box_coord_per_process, index_by_text_region_contours, slopes_per_each_subprocess]) + + def do_work_of_slopes_new(self, queue_of_all_params, boxes_text, textline_mask_tot_ea, contours_per_process, contours_par_per_process, indexes_r_con_per_pro, image_page_rotated, slope_deskew): + slopes_per_each_subprocess = [] - bounding_box_of_textregion_per_each_subprocess=[] - textlines_rectangles_per_each_subprocess=[] - contours_textregion_per_each_subprocess=[] - contours_textregion_par_per_each_subprocess=[] - all_box_coord_per_process=[] - index_by_text_region_contours=[] - slope_biggest=0 - + bounding_box_of_textregion_per_each_subprocess = [] + textlines_rectangles_per_each_subprocess = [] + contours_textregion_per_each_subprocess = [] + contours_textregion_par_per_each_subprocess = [] + all_box_coord_per_process = [] + index_by_text_region_contours = [] + slope_biggest = 0 - for mv in range(len(boxes_text)): - crop_img,crop_coor=self.crop_image_inside_box(boxes_text[mv],image_page_rotated) + crop_img,crop_coor=crop_image_inside_box(boxes_text[mv],image_page_rotated) #all_box_coord.append(crop_coor) @@ -1883,11024 +1274,2424 @@ class eynollah: if img_int_p.shape[0]/img_int_p.shape[1]<0.1: slopes_per_each_subprocess.append(0) - - slope_for_all=[slope_deskew][0] - - all_text_region_raw=textline_mask_tot_ea[boxes_text[mv][1]:boxes_text[mv][1]+boxes_text[mv][3] , boxes_text[mv][0]:boxes_text[mv][0]+boxes_text[mv][2] ] - ###cnt_clean_rot=self.textline_contours_postprocessing(all_text_region_raw,slopes[jj],contours_only_text_parent[jj],boxes_text[jj],slope_first) - cnt_clean_rot=self.textline_contours_postprocessing(all_text_region_raw,slope_for_all,contours_par_per_process[mv],boxes_text[mv],0) - + + slope_for_all = [slope_deskew][0] + + all_text_region_raw = textline_mask_tot_ea[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]] + ###cnt_clean_rot=textline_contours_postprocessing(all_text_region_raw,slopes[jj],contours_only_text_parent[jj],boxes_text[jj],slope_first) + cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contours_par_per_process[mv], boxes_text[mv], 0) + textlines_rectangles_per_each_subprocess.append(cnt_clean_rot) - + index_by_text_region_contours.append(indexes_r_con_per_pro[mv]) - #all_found_texline_polygons.append(cnt_clean_rot) - bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv] ) + # all_found_texline_polygons.append(cnt_clean_rot) + bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv]) else: - + try: - textline_con,hierachy=self.return_contours_of_image(img_int_p) - textline_con_fil=self.filter_contours_area_of_image(img_int_p,textline_con,hierachy,max_area=1,min_area=0.00008) - - y_diff_mean=self.find_contours_mean_y_diff(textline_con_fil) - - sigma_des=int( y_diff_mean * (4./40.0) ) - - - if sigma_des<1: - sigma_des=1 - - img_int_p[img_int_p>0]=1 - #slope_for_all=self.return_deskew_slope_new(img_int_p,sigma_des) - slope_for_all=self.return_deskew_slop(img_int_p,sigma_des) - - if abs(slope_for_all)<=0.5: - slope_for_all=[slope_deskew][0] - - + textline_con, hierachy = return_contours_of_image(img_int_p) + textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, hierachy, max_area=1, min_area=0.00008) + + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) + + sigma_des = int(y_diff_mean * (4.0 / 40.0)) + + if sigma_des < 1: + sigma_des = 1 + + img_int_p[img_int_p > 0] = 1 + # slope_for_all=self.return_deskew_slope_new(img_int_p,sigma_des) + slope_for_all = return_deskew_slop(img_int_p, sigma_des, dir_of_all=self.dir_of_all, f_name=self.f_name) + + if abs(slope_for_all) <= 0.5: + slope_for_all = [slope_deskew][0] + except: - slope_for_all=999 - - - ##slope_for_all=self.return_deskew_slop(img_int_p,sigma_des) - - if slope_for_all==999: - slope_for_all=[slope_deskew][0] + slope_for_all = 999 + + ##slope_for_all=return_deskew_slop(img_int_p,sigma_des, dir_of_all=self.dir_of_all, f_name=self.f_name) + + if slope_for_all == 999: + slope_for_all = [slope_deskew][0] ##if np.abs(slope_for_all)>32.5 and slope_for_all!=999: - ##slope_for_all=slope_biggest + ##slope_for_all=slope_biggest ##elif slope_for_all==999: - ##slope_for_all=slope_biggest + ##slope_for_all=slope_biggest slopes_per_each_subprocess.append(slope_for_all) - - - slope_first=0 - - mask_only_con_region=np.zeros(textline_mask_tot_ea.shape) - mask_only_con_region=cv2.fillPoly(mask_only_con_region,pts=[contours_par_per_process[mv]],color=(1,1,1)) - - - #plt.imshow(mask_only_con_region) - #plt.show() - all_text_region_raw=np.copy(textline_mask_tot_ea[boxes_text[mv][1]:boxes_text[mv][1]+boxes_text[mv][3] , boxes_text[mv][0]:boxes_text[mv][0]+boxes_text[mv][2] ]) - mask_only_con_region=mask_only_con_region[boxes_text[mv][1]:boxes_text[mv][1]+boxes_text[mv][3] , boxes_text[mv][0]:boxes_text[mv][0]+boxes_text[mv][2] ] - - + + slope_first = 0 + + mask_only_con_region = np.zeros(textline_mask_tot_ea.shape) + mask_only_con_region = cv2.fillPoly(mask_only_con_region, pts=[contours_par_per_process[mv]], color=(1, 1, 1)) + + # plt.imshow(mask_only_con_region) + # plt.show() + all_text_region_raw = np.copy(textline_mask_tot_ea[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]]) + mask_only_con_region = mask_only_con_region[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]] + ##plt.imshow(textline_mask_tot_ea) ##plt.show() ##plt.imshow(all_text_region_raw) ##plt.show() - - + ##plt.imshow(mask_only_con_region) ##plt.show() - all_text_region_raw[mask_only_con_region==0]=0 - ###cnt_clean_rot=self.textline_contours_postprocessing(all_text_region_raw,slopes[jj],contours_only_text_parent[jj],boxes_text[jj],slope_first) - cnt_clean_rot=self.textline_contours_postprocessing(all_text_region_raw,slope_for_all,contours_par_per_process[mv],boxes_text[mv],slope_first) - + all_text_region_raw[mask_only_con_region == 0] = 0 + ###cnt_clean_rot=textline_contours_postprocessing(all_text_region_raw,slopes[jj],contours_only_text_parent[jj],boxes_text[jj],slope_first) + cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contours_par_per_process[mv], boxes_text[mv], slope_first) + textlines_rectangles_per_each_subprocess.append(cnt_clean_rot) index_by_text_region_contours.append(indexes_r_con_per_pro[mv]) - #all_found_texline_polygons.append(cnt_clean_rot) - bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv] ) + # all_found_texline_polygons.append(cnt_clean_rot) + bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv]) - - - contours_textregion_per_each_subprocess.append(contours_per_process[mv]) contours_textregion_par_per_each_subprocess.append(contours_par_per_process[mv]) all_box_coord_per_process.append(crop_coor) - queue_of_all_params.put([slopes_per_each_subprocess, textlines_rectangles_per_each_subprocess, bounding_box_of_textregion_per_each_subprocess, contours_textregion_per_each_subprocess,contours_textregion_par_per_each_subprocess,all_box_coord_per_process,index_by_text_region_contours]) - + queue_of_all_params.put([slopes_per_each_subprocess, textlines_rectangles_per_each_subprocess, bounding_box_of_textregion_per_each_subprocess, contours_textregion_per_each_subprocess, contours_textregion_par_per_each_subprocess, all_box_coord_per_process, index_by_text_region_contours]) - def get_text_region_contours_and_boxes(self, image): - rgb_class_of_texts = (1, 1, 1) - mask_texts = np.all(image == rgb_class_of_texts, axis=-1) + def textline_contours(self, img, patches, scaler_h, scaler_w): - image = np.repeat(mask_texts[:, :, np.newaxis], 3, axis=2) * 255 - image = image.astype(np.uint8) + if patches: + model_textline, session_textline = self.start_new_session_and_model(self.model_textline_dir) + if not patches: + model_textline, session_textline = self.start_new_session_and_model(self.model_textline_dir_np) - image = cv2.morphologyEx(image, cv2.MORPH_OPEN, self.kernel) - image = cv2.morphologyEx(image, cv2.MORPH_CLOSE, self.kernel) + ##img = otsu_copy(img) + img = img.astype(np.uint8) + img_org = np.copy(img) + img_h = img_org.shape[0] + img_w = img_org.shape[1] - imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + img = resize_image(img_org, int(img_org.shape[0] * scaler_h), int(img_org.shape[1] * scaler_w)) - _, thresh = cv2.threshold(imgray, 0, 255, 0) + prediction_textline = self.do_prediction(patches, img, model_textline) - contours, hirarchy = cv2.findContours(thresh.copy(), cv2.cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - main_contours = self.filter_contours_area_of_image(thresh, contours, hirarchy, max_area=1, min_area=0.00001) - self.boxes = [] - - for jj in range(len(main_contours)): - x, y, w, h = cv2.boundingRect(main_contours[jj]) - self.boxes.append([x, y, w, h]) - + prediction_textline = resize_image(prediction_textline, img_h, img_w) - return main_contours - def boosting_headers_by_longshot_region_segmentation(self,textregion_pre_p,textregion_pre_np,img_only_text): - textregion_pre_p_org=np.copy(textregion_pre_p) - # 4 is drop capitals - headers_in_longshot= (textregion_pre_np[:,:,0]==2)*1 - #headers_in_longshot= ( (textregion_pre_np[:,:,0]==2) | (textregion_pre_np[:,:,0]==1) )*1 - textregion_pre_p[:,:,0][(headers_in_longshot[:,:] ==1) & (textregion_pre_p[:,:,0] !=4)]=2 - textregion_pre_p[:,:,0][textregion_pre_p[:,:,0]==1]=0 - #textregion_pre_p[:,:,0][( img_only_text[:,:]==1) & (textregion_pre_p[:,:,0]!=7) & (textregion_pre_p[:,:,0]!=2)]=1 # eralier it was so, but by this manner the drop capitals are alse deleted - textregion_pre_p[:,:,0][( img_only_text[:,:]==1) & (textregion_pre_p[:,:,0]!=7) & (textregion_pre_p[:,:,0]!=4) & (textregion_pre_p[:,:,0]!=2)]=1 - return textregion_pre_p - - def boosting_text_only_regions_by_header(self,textregion_pre_np,img_only_text): - result= (( img_only_text[:,:]==1) | (textregion_pre_np[:,:,0]==2) ) *1 - return result - - def get_all_image_patches_coordination(self, image_page): - self.all_box_coord=[] - for jk in range(len(self.boxes)): - _,crop_coor=self.crop_image_inside_box(self.boxes[jk],image_page) - self.all_box_coord.append(crop_coor) - + patches = False + prediction_textline_longshot = self.do_prediction(patches, img, model_textline) + + prediction_textline_longshot_true_size = resize_image(prediction_textline_longshot, img_h, img_w) + + # scaler_w=1.5 + # scaler_h=1.5 + # patches=True + # img= resize_image(img_org, int(img_org.shape[0]*scaler_h), int(img_org.shape[1]*scaler_w)) + + # prediction_textline_streched=self.do_prediction(patches,img,model_textline) + + # prediction_textline_streched= resize_image(prediction_textline_streched, img_h, img_w) - def textline_contours(self, img,patches,scaler_h,scaler_w): - - - - if patches: - model_textline, session_textline = self.start_new_session_and_model(self.model_textline_dir) - if not patches: - model_textline, session_textline = self.start_new_session_and_model(self.model_textline_dir_np) - - ##img = self.otsu_copy(img) - img = img.astype(np.uint8) - - img_org=np.copy(img) - img_h=img_org.shape[0] - img_w=img_org.shape[1] - - img= self.resize_image(img_org, int(img_org.shape[0]*scaler_h), int(img_org.shape[1]*scaler_w)) - - prediction_textline=self.do_prediction(patches,img,model_textline) - - prediction_textline= self.resize_image(prediction_textline, img_h, img_w) - - patches=False - prediction_textline_longshot=self.do_prediction(patches,img,model_textline) - - prediction_textline_longshot_true_size= self.resize_image(prediction_textline_longshot, img_h, img_w) - - - #scaler_w=1.5 - #scaler_h=1.5 - #patches=True - #img= self.resize_image(img_org, int(img_org.shape[0]*scaler_h), int(img_org.shape[1]*scaler_w)) - - #prediction_textline_streched=self.do_prediction(patches,img,model_textline) - - #prediction_textline_streched= self.resize_image(prediction_textline_streched, img_h, img_w) - ##plt.imshow(prediction_textline_streched[:,:,0]) ##plt.show() - - #sys.exit() + + # sys.exit() session_textline.close() del model_textline del session_textline del img del img_org - + gc.collect() - return prediction_textline[:,:,0],prediction_textline_longshot_true_size[:,:,0] - - def get_textlines_for_each_textregions(self, textline_mask_tot, boxes): - textline_mask_tot = cv2.erode(textline_mask_tot, self.kernel, iterations=1) - self.area_of_cropped = [] - self.all_text_region_raw = [] - for jk in range(len(boxes)): - crop_img, crop_coor = self.crop_image_inside_box(boxes[jk], - np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) - crop_img=crop_img.astype(np.uint8) - self.all_text_region_raw.append(crop_img[:, :, 0]) - self.area_of_cropped.append(crop_img.shape[0] * crop_img.shape[1]) - def seperate_lines_new_inside_teils2(self, img_patch, thetha): - + return prediction_textline[:, :, 0], prediction_textline_longshot_true_size[:, :, 0] - (h, w) = img_patch.shape[:2] - center = (w // 2, h // 2) - M = cv2.getRotationMatrix2D(center, -thetha, 1.0) - x_d = M[0, 2] - y_d = M[1, 2] + def do_work_of_slopes(self, q, poly, box_sub, boxes_per_process, textline_mask_tot, contours_per_process): + slope_biggest = 0 + slopes_sub = [] + boxes_sub_new = [] + poly_sub = [] + for mv in range(len(boxes_per_process)): - thetha = thetha / 180. * np.pi - rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) - #contour_text_interest_copy = contour_text_interest.copy() + crop_img, _ = crop_image_inside_box(boxes_per_process[mv], np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) + crop_img = crop_img[:, :, 0] + crop_img = cv2.erode(crop_img, self.kernel, iterations=2) - #x_cont = contour_text_interest[:, 0, 0] - #y_cont = contour_text_interest[:, 0, 1] - #x_cont = x_cont - np.min(x_cont) - #y_cont = y_cont - np.min(y_cont) + try: + textline_con, hierachy = return_contours_of_image(crop_img) + textline_con_fil = filter_contours_area_of_image(crop_img, textline_con, hierachy, max_area=1, min_area=0.0008) + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) - x_min_cont = 0 - x_max_cont = img_patch.shape[1] - y_min_cont = 0 - y_max_cont = img_patch.shape[0] + sigma_des = int(y_diff_mean * (4.0 / 40.0)) - xv = np.linspace(x_min_cont, x_max_cont, 1000) + if sigma_des < 1: + sigma_des = 1 - textline_patch_sum_along_width = img_patch.sum(axis=1) + crop_img[crop_img > 0] = 1 + slope_corresponding_textregion = return_deskew_slop(crop_img, sigma_des, dir_of_all=self.dir_of_all, f_name=self.f_name) - first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + except: + slope_corresponding_textregion = 999 - y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] - y_padded = np.zeros(len(y) + 40) - y_padded[20:len(y) + 20] = y - x = np.array(range(len(y))) + if slope_corresponding_textregion == 999: + slope_corresponding_textregion = slope_biggest + ##if np.abs(slope_corresponding_textregion)>12.5 and slope_corresponding_textregion!=999: + ##slope_corresponding_textregion=slope_biggest + ##elif slope_corresponding_textregion==999: + ##slope_corresponding_textregion=slope_biggest + slopes_sub.append(slope_corresponding_textregion) - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - if 1>0: + cnt_clean_rot = textline_contours_postprocessing(crop_img, slope_corresponding_textregion, contours_per_process[mv], boxes_per_process[mv]) - try: + poly_sub.append(cnt_clean_rot) + boxes_sub_new.append(boxes_per_process[mv]) - y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) - y_padded_up_to_down_e=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) - y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e - y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) - + q.put(slopes_sub) + poly.put(poly_sub) + box_sub.put(boxes_sub_new) - peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) - peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) - neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + def write_into_page_xml_full(self, contours, contours_h, page_coord, dir_of_image, order_of_texts, id_of_texts, all_found_texline_polygons, all_found_texline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals): - arg_neg_must_be_deleted= np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3 ] - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - + found_polygons_text_region = contours + found_polygons_text_region_h = contours_h - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - + # create the file structure + data = ET.Element("PcGts") - peaks_new=peaks_e[:] - peaks_neg_new=peaks_neg_e[:] - - clusters_to_be_deleted=[] - if len(arg_diff_cluster)>0: - - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - - - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] - - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - - - else: - peaks_new_tot=peaks_e[:] + data.set("xmlns", "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15") + data.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") + data.set("xsi:schemaLocation", "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15") + metadata = ET.SubElement(data, "Metadata") - textline_con,hierachy=self.return_contours_of_image(img_patch) - textline_con_fil=self.filter_contours_area_of_image(img_patch,textline_con,hierachy,max_area=1,min_area=0.0008) - y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) + author = ET.SubElement(metadata, "Creator") + author.text = "SBB_QURATOR" - sigma_gaus=int( y_diff_mean * (7./40.0) ) - #print(sigma_gaus,'sigma_gaus') - except: - sigma_gaus=12 - if sigma_gaus<3: - sigma_gaus=3 - #print(sigma_gaus,'sigma') - - - y_padded_smoothed= gaussian_filter1d(y_padded, sigma_gaus) - y_padded_up_to_down=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded=np.zeros(len(y_padded_up_to_down)+40) - y_padded_up_to_down_padded[20:len(y_padded_up_to_down)+20]=y_padded_up_to_down - y_padded_up_to_down_padded= gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) - + created = ET.SubElement(metadata, "Created") + created.text = "2019-06-17T18:15:12" - peaks, _ = find_peaks(y_padded_smoothed, height=0) - peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) - - - peaks_new=peaks[:] - peaks_neg_new=peaks_neg[:] - - try: - neg_peaks_max=np.max(y_padded_smoothed[peaks]) - + changetime = ET.SubElement(metadata, "LastChange") + changetime.text = "2019-06-17T18:15:12" - arg_neg_must_be_deleted= np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.24 ] + page = ET.SubElement(data, "Page") + page.set("imageFilename", self.image_dir) + page.set("imageHeight", str(self.height_org)) + page.set("imageWidth", str(self.width_org)) + page.set("type", "content") + page.set("readingDirection", "left-to-right") + page.set("textLineOrder", "top-to-bottom") - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - + page_print_sub = ET.SubElement(page, "PrintSpace") + coord_page = ET.SubElement(page_print_sub, "Coords") + points_page_print = "" - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - + for lmm in range(len(self.cont_page[0])): + if len(self.cont_page[0][lmm]) == 2: + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0]) / self.scale_x)) + points_page_print = points_page_print + "," + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][1]) / self.scale_y)) + else: + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0][0]) / self.scale_x)) + points_page_print = points_page_print + "," + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0][1]) / self.scale_y)) + if lmm < (len(self.cont_page[0]) - 1): + points_page_print = points_page_print + " " + coord_page.set("points", points_page_print) - clusters_to_be_deleted=[] - + if len(contours) > 0: + region_order = ET.SubElement(page, "ReadingOrder") + region_order_sub = ET.SubElement(region_order, "OrderedGroup") - if len(arg_diff_cluster)>=2 and len(arg_diff_cluster)>0: - - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - elif len(arg_neg_must_be_deleted)>=2 and len(arg_diff_cluster)==0: - clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - - - if len(arg_neg_must_be_deleted)==1: - clusters_to_be_deleted.append(arg_neg_must_be_deleted) - - - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]]] - - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - - #plt.plot(y_padded_up_to_down_padded) - #plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') - #plt.show() - - #plt.plot(y_padded_up_to_down_padded) - #plt.plot(peaks_neg_new,y_padded_up_to_down_padded[peaks_neg_new],'*') - #plt.show() - - #plt.plot(y_padded_smoothed) - #plt.plot(peaks,y_padded_smoothed[peaks],'*') - #plt.show() - - #plt.plot(y_padded_smoothed) - #plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') - #plt.show() - - peaks=peaks_new_tot[:] - peaks_neg=peaks_neg_new[:] - except: - pass - - + region_order_sub.set("id", "ro357564684568544579089") - + # args_sort=order_of_texts + for vj in order_of_texts: + name = "coord_text_" + str(vj) + name = ET.SubElement(region_order_sub, "RegionRefIndexed") + name.set("index", str(order_of_texts[vj])) + name.set("regionRef", id_of_texts[vj]) + id_of_marginalia = [] + indexer_region = len(contours) + len(contours_h) + for vm in range(len(found_polygons_marginals)): + id_of_marginalia.append("r" + str(indexer_region)) - - - else: - peaks_new_tot=peaks[:] - peaks=peaks_new_tot[:] - peaks_neg=peaks_neg_new[:] - - mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks=np.std(y_padded_smoothed[peaks]) - peaks_values=y_padded_smoothed[peaks] - + name = "coord_text_" + str(indexer_region) + name = ET.SubElement(region_order_sub, "RegionRefIndexed") + name.set("index", str(indexer_region)) + name.set("regionRef", "r" + str(indexer_region)) + indexer_region += 1 - ###peaks_neg = peaks_neg - 20 - 20 - ###peaks = peaks - 20 - peaks_neg_true=peaks_neg[:] - peaks_pos_true=peaks[:] - - if len(peaks_neg_true)>0: - peaks_neg_true=np.array(peaks_neg_true) + id_indexer = 0 + id_indexer_l = 0 - peaks_neg_true = peaks_neg_true - 20 - 20 - - - #print(peaks_neg_true) - for i in range(len(peaks_neg_true)): - img_patch[peaks_neg_true[i]-6:peaks_neg_true[i]+6,:]=0 - - - else: - pass - - if len(peaks_pos_true)>0: - peaks_pos_true=np.array(peaks_pos_true) - peaks_pos_true = peaks_pos_true - 20 - - for i in range(len(peaks_pos_true)): - ##img_patch[peaks_pos_true[i]-8:peaks_pos_true[i]+8,:]=1 - img_patch[peaks_pos_true[i]-6:peaks_pos_true[i]+6,:]=1 - else: - pass - kernel = np.ones((5,5),np.uint8) + for mm in range(len(found_polygons_text_region)): + textregion = ET.SubElement(page, "TextRegion") - #img_patch = cv2.erode(img_patch,kernel,iterations = 3) - #######################img_patch = cv2.erode(img_patch,kernel,iterations = 2) - img_patch = cv2.erode(img_patch,kernel,iterations = 1) - return img_patch + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 - def seperate_lines_new_inside_teils(self, img_path, thetha): - (h, w) = img_path.shape[:2] - center = (w // 2, h // 2) - M = cv2.getRotationMatrix2D(center, -thetha, 1.0) - x_d = M[0, 2] - y_d = M[1, 2] + textregion.set("type", "paragraph") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") - thetha = thetha / 180. * np.pi - rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + points_co = "" + for lmm in range(len(found_polygons_text_region[mm])): + if len(found_polygons_text_region[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) + if lmm < (len(found_polygons_text_region[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) - x_min_cont = 0 - x_max_cont = img_path.shape[1] - y_min_cont = 0 - y_max_cont = img_path.shape[0] + for j in range(len(all_found_texline_polygons[mm])): - xv = np.linspace(x_min_cont, x_max_cont, 1000) + textline = ET.SubElement(textregion, "TextLine") - mada_n = img_path.sum(axis=1) - - ##plt.plot(mada_n) - ##plt.show() + textline.set("id", "l" + str(id_indexer_l)) - first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + id_indexer_l += 1 - y = mada_n[:] # [first_nonzero:last_nonzero] - y_help = np.zeros(len(y) + 40) - y_help[20:len(y) + 20] = y - x = np.array(range(len(y))) + coord = ET.SubElement(textline, "Coords") - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - if len(peaks_real)<=2 and len(peaks_real)>1: - sigma_gaus=10 - else: - sigma_gaus=5 - - - z= gaussian_filter1d(y_help, sigma_gaus) - zneg_rev=-y_help+np.max(y_help) - zneg=np.zeros(len(zneg_rev)+40) - zneg[20:len(zneg_rev)+20]=zneg_rev - zneg= gaussian_filter1d(zneg, sigma_gaus) - - peaks, _ = find_peaks(z, height=0) - peaks_neg, _ = find_peaks(zneg, height=0) - - for nn in range(len(peaks_neg)): - if peaks_neg[nn]>len(z)-1: - peaks_neg[nn]=len(z)-1 - if peaks_neg[nn]<0: - peaks_neg[nn]=0 - - diff_peaks=np.abs( np.diff(peaks_neg) ) - - cut_off=20 - peaks_neg_true=[] - forest=[] - - for i in range(len(peaks_neg)): - if i==0: - forest.append(peaks_neg[i]) - if i<(len(peaks_neg)-1): - if diff_peaks[i]<=cut_off: - forest.append(peaks_neg[i+1]) - if diff_peaks[i]>cut_off: - #print(forest[np.argmin(z[forest]) ] ) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - forest=[] - forest.append(peaks_neg[i+1]) - if i==(len(peaks_neg)-1): - #print(print(forest[np.argmin(z[forest]) ] )) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - - - diff_peaks_pos=np.abs( np.diff(peaks) ) - - cut_off=20 - peaks_pos_true=[] - forest=[] - - for i in range(len(peaks)): - if i==0: - forest.append(peaks[i]) - if i<(len(peaks)-1): - if diff_peaks_pos[i]<=cut_off: - forest.append(peaks[i+1]) - if diff_peaks_pos[i]>cut_off: - #print(forest[np.argmin(z[forest]) ] ) - if not self.isNaN(forest[np.argmax(z[forest]) ]): - peaks_pos_true.append(forest[np.argmax(z[forest]) ]) - forest=[] - forest.append(peaks[i+1]) - if i==(len(peaks)-1): - #print(print(forest[np.argmin(z[forest]) ] )) - if not self.isNaN(forest[np.argmax(z[forest]) ]): - peaks_pos_true.append(forest[np.argmax(z[forest]) ]) - - #print(len(peaks_neg_true) ,len(peaks_pos_true) ,'lensss') - - if len(peaks_neg_true)>0: - peaks_neg_true=np.array(peaks_neg_true) - """ - #plt.figure(figsize=(40,40)) - #plt.subplot(1,2,1) - #plt.title('Textline segmentation von Textregion') - #plt.imshow(img_path) - #plt.xlabel('X') - #plt.ylabel('Y') - #plt.subplot(1,2,2) - #plt.title('Dichte entlang X') - #base = pyplot.gca().transData - #rot = transforms.Affine2D().rotate_deg(90) - #plt.plot(zneg,np.array(range(len(zneg)))) - #plt.plot(zneg[peaks_neg_true],peaks_neg_true,'*') - #plt.gca().invert_yaxis() - - #plt.xlabel('Dichte') - #plt.ylabel('Y') - ##plt.plot([0,len(y)], [grenze,grenze]) - #plt.show() - """ - peaks_neg_true = peaks_neg_true - 20 - 20 - - - #print(peaks_neg_true) - for i in range(len(peaks_neg_true)): - img_path[peaks_neg_true[i]-6:peaks_neg_true[i]+6,:]=0 - - - else: - pass - - if len(peaks_pos_true)>0: - peaks_pos_true=np.array(peaks_pos_true) - peaks_pos_true = peaks_pos_true - 20 - - for i in range(len(peaks_pos_true)): - img_path[peaks_pos_true[i]-8:peaks_pos_true[i]+8,:]=1 - else: - pass - kernel = np.ones((5,5),np.uint8) + texteq = ET.SubElement(textline, "TextEquiv") - #img_path = cv2.erode(img_path,kernel,iterations = 3) - img_path = cv2.erode(img_path,kernel,iterations = 2) - return img_path - - - - def seperate_lines_new(self, img_path, thetha,num_col): - - if num_col==1: - num_patches=int(img_path.shape[1]/200.) - else: - num_patches=int(img_path.shape[1]/100.) - #num_patches=int(img_path.shape[1]/200.) - if num_patches==0: - num_patches=1 - (h, w) = img_path.shape[:2] - center = (w // 2, h // 2) - M = cv2.getRotationMatrix2D(center, -thetha, 1.0) - x_d = M[0, 2] - y_d = M[1, 2] + uni = ET.SubElement(texteq, "Unicode") + uni.text = " " - thetha = thetha / 180. * np.pi - rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + # points = ET.SubElement(coord, 'Points') + points_co = "" + for l in range(len(all_found_texline_polygons[mm][j])): + # point = ET.SubElement(coord, 'Point') + + if not self.curved_line: + # point.set('x',str(found_polygons[j][l][0])) + # point.set('y',str(found_polygons[j][l][1])) + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + + if (self.curved_line) and np.abs(slopes[mm]) <= 45: + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + page_coord[0]) / self.scale_y)) + elif (self.curved_line) and np.abs(slopes[mm]) > 45: + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) - x_min_cont = 0 - x_max_cont = img_path.shape[1] - y_min_cont = 0 - y_max_cont = img_path.shape[0] + if l < (len(all_found_texline_polygons[mm][j]) - 1): + points_co = points_co + " " + # print(points_co) + coord.set("points", points_co) - xv = np.linspace(x_min_cont, x_max_cont, 1000) + texteqreg = ET.SubElement(textregion, "TextEquiv") - mada_n = img_path.sum(axis=1) - - ##plt.plot(mada_n) - ##plt.show() - first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) - - y = mada_n[:] # [first_nonzero:last_nonzero] - y_help = np.zeros(len(y) + 40) - y_help[20:len(y) + 20] = y - x = np.array(range(len(y))) - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - if len(peaks_real)<=2 and len(peaks_real)>1: - sigma_gaus=10 - else: - sigma_gaus=6 - - - z= gaussian_filter1d(y_help, sigma_gaus) - zneg_rev=-y_help+np.max(y_help) - zneg=np.zeros(len(zneg_rev)+40) - zneg[20:len(zneg_rev)+20]=zneg_rev - zneg= gaussian_filter1d(zneg, sigma_gaus) - - peaks, _ = find_peaks(z, height=0) - peaks_neg, _ = find_peaks(zneg, height=0) - - - for nn in range(len(peaks_neg)): - if peaks_neg[nn]>len(z)-1: - peaks_neg[nn]=len(z)-1 - if peaks_neg[nn]<0: - peaks_neg[nn]=0 - - diff_peaks=np.abs( np.diff(peaks_neg) ) - cut_off=20 - peaks_neg_true=[] - forest=[] - - for i in range(len(peaks_neg)): - if i==0: - forest.append(peaks_neg[i]) - if i<(len(peaks_neg)-1): - if diff_peaks[i]<=cut_off: - forest.append(peaks_neg[i+1]) - if diff_peaks[i]>cut_off: - #print(forest[np.argmin(z[forest]) ] ) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - #print(len(z),forest) - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - forest=[] - forest.append(peaks_neg[i+1]) - if i==(len(peaks_neg)-1): - #print(print(forest[np.argmin(z[forest]) ] )) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - - - - peaks_neg_true=np.array(peaks_neg_true) - - """ - #plt.figure(figsize=(40,40)) - #plt.subplot(1,2,1) - #plt.title('Textline segmentation von Textregion') - #plt.imshow(img_path) - #plt.xlabel('X') - #plt.ylabel('Y') - #plt.subplot(1,2,2) - #plt.title('Dichte entlang X') - #base = pyplot.gca().transData - #rot = transforms.Affine2D().rotate_deg(90) - #plt.plot(zneg,np.array(range(len(zneg)))) - #plt.plot(zneg[peaks_neg_true],peaks_neg_true,'*') - #plt.gca().invert_yaxis() - - #plt.xlabel('Dichte') - #plt.ylabel('Y') - ##plt.plot([0,len(y)], [grenze,grenze]) - #plt.show() - """ - - peaks_neg_true = peaks_neg_true - 20 - 20 - peaks = peaks - 20 - - #dis_up=peaks_neg_true[14]-peaks_neg_true[0] - #dis_down=peaks_neg_true[18]-peaks_neg_true[14] - - img_patch_ineterst=img_path[:,:]#[peaks_neg_true[14]-dis_up:peaks_neg_true[15]+dis_down ,:] - - ##plt.imshow(img_patch_ineterst) - ##plt.show() - - - - - length_x=int(img_path.shape[1]/float(num_patches)) - margin = int(0.04 * length_x) - - width_mid = length_x - 2 * margin + unireg = ET.SubElement(texteqreg, "Unicode") + unireg.text = " " + print(len(contours_h)) + if len(contours_h) > 0: + for mm in range(len(found_polygons_text_region_h)): + textregion = ET.SubElement(page, "TextRegion") + try: + id_indexer = id_indexer + id_indexer_l = id_indexer_l + except: + id_indexer = 0 + id_indexer_l = 0 + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 + textregion.set("type", "header") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") + points_co = "" + for lmm in range(len(found_polygons_text_region_h[mm])): + if len(found_polygons_text_region_h[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_text_region_h[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region_h[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_text_region_h[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region_h[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - nxf = img_path.shape[1] / float(width_mid) + if lmm < (len(found_polygons_text_region_h[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) - if nxf > int(nxf): - nxf = int(nxf) + 1 - else: - nxf = int(nxf) + for j in range(len(all_found_texline_polygons_h[mm])): - slopes_tile_wise=[] - for i in range(nxf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x - elif i > 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x + textline = ET.SubElement(textregion, "TextLine") + textline.set("id", "l" + str(id_indexer_l)) - if index_x_u > img_path.shape[1]: - index_x_u = img_path.shape[1] - index_x_d = img_path.shape[1] - length_x + id_indexer_l += 1 - - + coord = ET.SubElement(textline, "Coords") - #img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - img_xline=img_patch_ineterst[:,index_x_d:index_x_u] - - - sigma=2 - try: - slope_xline=self.return_deskew_slop(img_xline,sigma) - except: - slope_xline=0 - slopes_tile_wise.append(slope_xline) - #print(slope_xline,'xlineeee') - img_line_rotated=self.rotate_image(img_xline,slope_xline) - img_line_rotated[:,:][img_line_rotated[:,:]!=0]=1 - - """ - - xline=np.linspace(0,img_path.shape[1],nx) - slopes_tile_wise=[] - - for ui in range( nx-1 ): - img_xline=img_patch_ineterst[:,int(xline[ui]):int(xline[ui+1])] - - - ##plt.imshow(img_xline) - ##plt.show() - - sigma=3 - try: - slope_xline=self.return_deskew_slop(img_xline,sigma) - except: - slope_xline=0 - slopes_tile_wise.append(slope_xline) - print(slope_xline,'xlineeee') - img_line_rotated=self.rotate_image(img_xline,slope_xline) - - ##plt.imshow(img_line_rotated) - ##plt.show() - """ - - #dis_up=peaks_neg_true[14]-peaks_neg_true[0] - #dis_down=peaks_neg_true[18]-peaks_neg_true[14] - - img_patch_ineterst=img_path[:,:]#[peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] - - img_patch_ineterst_revised=np.zeros(img_patch_ineterst.shape) - + texteq = ET.SubElement(textline, "TextEquiv") - for i in range(nxf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x - elif i > 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x + uni = ET.SubElement(texteq, "Unicode") + uni.text = " " + # points = ET.SubElement(coord, 'Points') - if index_x_u > img_path.shape[1]: - index_x_u = img_path.shape[1] - index_x_d = img_path.shape[1] - length_x - - img_xline=img_patch_ineterst[:,index_x_d:index_x_u] - - img_int=np.zeros((img_xline.shape[0],img_xline.shape[1])) - img_int[:,:]=img_xline[:,:]#img_patch_org[:,:,0] + points_co = "" + for l in range(len(all_found_texline_polygons_h[mm][j])): + # point = ET.SubElement(coord, 'Point') + + if not self.curved_line: + # point.set('x',str(found_polygons[j][l][0])) + # point.set('y',str(found_polygons[j][l][1])) + if len(all_found_texline_polygons_h[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][0] + all_box_coord_h[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][1] + all_box_coord_h[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][0][0] + all_box_coord_h[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][0][1] + all_box_coord_h[mm][0] + page_coord[0]) / self.scale_y)) + + if self.curved_line: + if len(all_found_texline_polygons_h[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_h[mm][j][l][0][1] + page_coord[0]) / self.scale_y)) - img_resized=np.zeros((int( img_int.shape[0]*(1.2) ) , int( img_int.shape[1]*(3) ) )) - - img_resized[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ]=img_int[:,:] - ##plt.imshow(img_xline) - ##plt.show() - img_line_rotated=self.rotate_image(img_resized,slopes_tile_wise[i]) - img_line_rotated[:,:][img_line_rotated[:,:]!=0]=1 - - - - img_patch_seperated=self.seperate_lines_new_inside_teils(img_line_rotated,0) - - ##plt.imshow(img_patch_seperated) - ##plt.show() - img_patch_seperated_returned=self.rotate_image(img_patch_seperated,-slopes_tile_wise[i]) - img_patch_seperated_returned[:,:][img_patch_seperated_returned[:,:]!=0]=1 - - img_patch_seperated_returned_true_size=img_patch_seperated_returned[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ] - - img_patch_seperated_returned_true_size = img_patch_seperated_returned_true_size[:, margin:length_x - margin] - img_patch_ineterst_revised[:,index_x_d + margin:index_x_u - margin]=img_patch_seperated_returned_true_size + if l < (len(all_found_texline_polygons_h[mm][j]) - 1): + points_co = points_co + " " + # print(points_co) + coord.set("points", points_co) - - """ - for ui in range( nx-1 ): - img_xline=img_patch_ineterst[:,int(xline[ui]):int(xline[ui+1])] - - - img_int=np.zeros((img_xline.shape[0],img_xline.shape[1])) - img_int[:,:]=img_xline[:,:]#img_patch_org[:,:,0] + texteqreg = ET.SubElement(textregion, "TextEquiv") - img_resized=np.zeros((int( img_int.shape[0]*(1.2) ) , int( img_int.shape[1]*(3) ) )) - - img_resized[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ]=img_int[:,:] - ##plt.imshow(img_xline) - ##plt.show() - img_line_rotated=self.rotate_image(img_resized,slopes_tile_wise[ui]) - - - #img_patch_seperated=self.seperate_lines_new_inside_teils(img_line_rotated,0) - - img_patch_seperated=self.seperate_lines_new_inside_teils(img_line_rotated,0) - - img_patch_seperated_returned=self.rotate_image(img_patch_seperated,-slopes_tile_wise[ui]) - ##plt.imshow(img_patch_seperated) - ##plt.show() - print(img_patch_seperated_returned.shape) - #plt.imshow(img_patch_seperated_returned[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ]) - #plt.show() - - img_patch_ineterst_revised[:,int(xline[ui]):int(xline[ui+1])]=img_patch_seperated_returned[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ] - - - """ - - #print(img_patch_ineterst_revised.shape,np.unique(img_patch_ineterst_revised)) - ##plt.imshow(img_patch_ineterst_revised) - ##plt.show() - return img_patch_ineterst_revised - - def seperate_lines_new2(self, img_path, thetha,num_col,slope_region): - - if num_col==1: - num_patches=int(img_path.shape[1]/200.) - else: - num_patches=int(img_path.shape[1]/140.) - #num_patches=int(img_path.shape[1]/200.) - if num_patches==0: - num_patches=1 + unireg = ET.SubElement(texteqreg, "Unicode") + unireg.text = " " - - img_patch_ineterst=img_path[:,:]#[peaks_neg_true[14]-dis_up:peaks_neg_true[15]+dis_down ,:] - - #plt.imshow(img_patch_ineterst) - #plt.show() - - - - - length_x=int(img_path.shape[1]/float(num_patches)) - #margin = int(0.04 * length_x) just recently this was changed because it break lines into 2 - margin = int(0.04 * length_x) - #print(margin,'margin') - #if margin<=4: - #margin = int(0.08 * length_x) - - #margin=0 - - width_mid = length_x - 2 * margin + if len(found_polygons_drop_capitals) > 0: + id_indexer = len(contours_h) + len(contours) + len(found_polygons_marginals) + for mm in range(len(found_polygons_drop_capitals)): + textregion = ET.SubElement(page, "TextRegion") - + # id_indexer_l=id_indexer_l + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 + textregion.set("type", "drop-capital") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") - nxf = img_path.shape[1] / float(width_mid) + points_co = "" + for lmm in range(len(found_polygons_drop_capitals[mm])): - if nxf > int(nxf): - nxf = int(nxf) + 1 - else: - nxf = int(nxf) + if len(found_polygons_drop_capitals[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_drop_capitals[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_drop_capitals[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_drop_capitals[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_drop_capitals[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - slopes_tile_wise=[] - for i in range(nxf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x - elif i > 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x + if lmm < (len(found_polygons_drop_capitals[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) + ##for j in range(len(all_found_texline_polygons_h[mm])): - if index_x_u > img_path.shape[1]: - index_x_u = img_path.shape[1] - index_x_d = img_path.shape[1] - length_x + ##textline=ET.SubElement(textregion, 'TextLine') + + ##textline.set('id','l'+str(id_indexer_l)) + + ##id_indexer_l+=1 + + ##coord = ET.SubElement(textline, 'Coords') + + ##texteq=ET.SubElement(textline, 'TextEquiv') + + ##uni=ET.SubElement(texteq, 'Unicode') + ##uni.text = ' ' + + ###points = ET.SubElement(coord, 'Points') + + ##points_co='' + ##for l in range(len(all_found_texline_polygons_h[mm][j])): + ###point = ET.SubElement(coord, 'Point') + + ##if not curved_line: + ###point.set('x',str(found_polygons[j][l][0])) + ###point.set('y',str(found_polygons[j][l][1])) + ##if len(all_found_texline_polygons_h[mm][j][l])==2: + ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] + ##+all_box_coord_h[mm][2]+page_coord[2])/self.scale_x) ) + ##points_co=points_co+',' + ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] + ##+all_box_coord_h[mm][0]+page_coord[0])/self.scale_y) ) + ##else: + ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] + ##+all_box_coord_h[mm][2]+page_coord[2])/self.scale_x ) ) + ##points_co=points_co+',' + ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] + ##+all_box_coord_h[mm][0]+page_coord[0])/self.scale_y) ) + + ##if curved_line: + ##if len(all_found_texline_polygons_h[mm][j][l])==2: + ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] + ##+page_coord[2])/self.scale_x) ) + ##points_co=points_co+',' + ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] + ##+page_coord[0])/self.scale_y) ) + ##else: + ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] + ##+page_coord[2])/self.scale_x ) ) + ##points_co=points_co+',' + ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] + ##+page_coord[0])/self.scale_y) ) + + ##if l<(len(all_found_texline_polygons_h[mm][j])-1): + ##points_co=points_co+' ' + ###print(points_co) + ####coord.set('points',points_co) + + texteqreg = ET.SubElement(textregion, "TextEquiv") + + unireg = ET.SubElement(texteqreg, "Unicode") + unireg.text = " " - - + try: - #img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - img_xline=img_patch_ineterst[:,index_x_d:index_x_u] - - - sigma=2 try: - slope_xline=self.return_deskew_slop(img_xline,sigma) + ###id_indexer=id_indexer + id_indexer_l = id_indexer_l except: - slope_xline=0 - - if abs(slope_region)<25 and abs(slope_xline)>25: - slope_xline=[slope_region][0] - #if abs(slope_region)>70 and abs(slope_xline)<25: - #slope_xline=[slope_region][0] - slopes_tile_wise.append(slope_xline) - #print(slope_xline,'xlineeee') - img_line_rotated=self.rotate_image(img_xline,slope_xline) - img_line_rotated[:,:][img_line_rotated[:,:]!=0]=1 - - - #print(slopes_tile_wise,'slopes_tile_wise') - img_patch_ineterst=img_path[:,:]#[peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] - - img_patch_ineterst_revised=np.zeros(img_patch_ineterst.shape) - - - for i in range(nxf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x - elif i > 0: - index_x_d = i * width_mid - index_x_u = index_x_d + length_x - + ###id_indexer=0 + id_indexer_l = 0 + for mm in range(len(found_polygons_marginals)): + textregion = ET.SubElement(page, "TextRegion") - if index_x_u > img_path.shape[1]: - index_x_u = img_path.shape[1] - index_x_d = img_path.shape[1] - length_x - - img_xline=img_patch_ineterst[:,index_x_d:index_x_u] - - img_int=np.zeros((img_xline.shape[0],img_xline.shape[1])) - img_int[:,:]=img_xline[:,:]#img_patch_org[:,:,0] + textregion.set("id", id_of_marginalia[mm]) - img_resized=np.zeros((int( img_int.shape[0]*(1.2) ) , int( img_int.shape[1]*(3) ) )) - - img_resized[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ]=img_int[:,:] - #plt.imshow(img_xline) - #plt.show() - img_line_rotated=self.rotate_image(img_resized,slopes_tile_wise[i]) - img_line_rotated[:,:][img_line_rotated[:,:]!=0]=1 - - - - img_patch_seperated=self.seperate_lines_new_inside_teils2(img_line_rotated,0) - + textregion.set("type", "marginalia") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") - img_patch_seperated_returned=self.rotate_image(img_patch_seperated,-slopes_tile_wise[i]) - img_patch_seperated_returned[:,:][img_patch_seperated_returned[:,:]!=0]=1 - - img_patch_seperated_returned_true_size=img_patch_seperated_returned[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ] - - img_patch_seperated_returned_true_size = img_patch_seperated_returned_true_size[:, margin:length_x - margin] - img_patch_ineterst_revised[:,index_x_d + margin:index_x_u - margin]=img_patch_seperated_returned_true_size + points_co = "" + for lmm in range(len(found_polygons_marginals[mm])): + if len(found_polygons_marginals[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - - #plt.imshow(img_patch_ineterst_revised) - #plt.show() - return img_patch_ineterst_revised + if lmm < (len(found_polygons_marginals[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) - def seperate_lines(self, img_patch, contour_text_interest, thetha,x_help,y_help): - - - (h, w) = img_patch.shape[:2] - center = (w // 2, h // 2) - M = cv2.getRotationMatrix2D(center, -thetha, 1.0) - x_d = M[0, 2] - y_d = M[1, 2] - - thetha = thetha / 180. * np.pi - rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) - contour_text_interest_copy = contour_text_interest.copy() + for j in range(len(all_found_texline_polygons_marginals[mm])): - x_cont = contour_text_interest[:, 0, 0] - y_cont = contour_text_interest[:, 0, 1] - x_cont = x_cont - np.min(x_cont) - y_cont = y_cont - np.min(y_cont) + textline = ET.SubElement(textregion, "TextLine") - x_min_cont = 0 - x_max_cont = img_patch.shape[1] - y_min_cont = 0 - y_max_cont = img_patch.shape[0] + textline.set("id", "l" + str(id_indexer_l)) - xv = np.linspace(x_min_cont, x_max_cont, 1000) + id_indexer_l += 1 - textline_patch_sum_along_width = img_patch.sum(axis=1) + coord = ET.SubElement(textline, "Coords") - first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + texteq = ET.SubElement(textline, "TextEquiv") - y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] - y_padded = np.zeros(len(y) + 40) - y_padded[20:len(y) + 20] = y - x = np.array(range(len(y))) + uni = ET.SubElement(texteq, "Unicode") + uni.text = " " - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - - if 1>0: + # points = ET.SubElement(coord, 'Points') - try: + points_co = "" + for l in range(len(all_found_texline_polygons_marginals[mm][j])): + # point = ET.SubElement(coord, 'Point') + + if not self.curved_line: + # point.set('x',str(found_polygons[j][l][0])) + # point.set('y',str(found_polygons[j][l][1])) + if len(all_found_texline_polygons_marginals[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0] + all_box_coord_marginals[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][1] + all_box_coord_marginals[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][0] + all_box_coord_marginals[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][1] + all_box_coord_marginals[mm][0] + page_coord[0]) / self.scale_y)) + + if self.curved_line: + if len(all_found_texline_polygons_marginals[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][1] + page_coord[0]) / self.scale_y)) - y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) - y_padded_up_to_down_e=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) - y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e - y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) - + if l < (len(all_found_texline_polygons_marginals[mm][j]) - 1): + points_co = points_co + " " + # print(points_co) + coord.set("points", points_co) - peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) - peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) - neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + texteqreg = ET.SubElement(textregion, "TextEquiv") - arg_neg_must_be_deleted= np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3 ] - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - + unireg = ET.SubElement(texteqreg, "Unicode") + unireg.text = " " + except: + pass - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - + try: + id_indexer = len(contours_h) + len(contours) + len(found_polygons_marginals) + len(found_polygons_drop_capitals) + for mm in range(len(found_polygons_text_region_img)): + textregion = ET.SubElement(page, "ImageRegion") - peaks_new=peaks_e[:] - peaks_neg_new=peaks_neg_e[:] - - clusters_to_be_deleted=[] - if len(arg_diff_cluster)>0: - - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - - - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] - - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - - - else: - peaks_new_tot=peaks_e[:] + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 + coord_text = ET.SubElement(textregion, "Coords") - textline_con,hierachy=self.return_contours_of_image(img_patch) - textline_con_fil=self.filter_contours_area_of_image(img_patch,textline_con,hierachy,max_area=1,min_area=0.0008) - y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) + points_co = "" + for lmm in range(len(found_polygons_text_region_img[mm])): - sigma_gaus=int( y_diff_mean * (7./40.0) ) - #print(sigma_gaus,'sigma_gaus') - except: - sigma_gaus=12 - if sigma_gaus<3: - sigma_gaus=3 - #print(sigma_gaus,'sigma') - - - y_padded_smoothed= gaussian_filter1d(y_padded, sigma_gaus) - y_padded_up_to_down=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded=np.zeros(len(y_padded_up_to_down)+40) - y_padded_up_to_down_padded[20:len(y_padded_up_to_down)+20]=y_padded_up_to_down - y_padded_up_to_down_padded= gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) - + if len(found_polygons_text_region_img[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_text_region_img[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region_img[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_text_region_img[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region_img[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - peaks, _ = find_peaks(y_padded_smoothed, height=0) - peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) - - + if lmm < (len(found_polygons_text_region_img[mm]) - 1): + points_co = points_co + " " + coord_text.set("points", points_co) + except: + pass - try: - neg_peaks_max=np.max(y_padded_smoothed[peaks]) - + for mm in range(len(found_polygons_tables)): + textregion = ET.SubElement(page, "TableRegion") - arg_neg_must_be_deleted= np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.42 ] + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 + coord_text = ET.SubElement(textregion, "Coords") - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - + points_co = "" + for lmm in range(len(found_polygons_tables[mm])): - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - except: - arg_neg_must_be_deleted=[] - arg_diff_cluster=[] - - - try: - peaks_new=peaks[:] - peaks_neg_new=peaks_neg[:] - clusters_to_be_deleted=[] - + if len(found_polygons_tables[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_tables[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_tables[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_tables[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_tables[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - if len(arg_diff_cluster)>=2 and len(arg_diff_cluster)>0: - - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - elif len(arg_neg_must_be_deleted)>=2 and len(arg_diff_cluster)==0: - clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - - - - if len(arg_neg_must_be_deleted)==1: - clusters_to_be_deleted.append(arg_neg_must_be_deleted) - + if lmm < (len(found_polygons_tables[mm]) - 1): + points_co = points_co + " " - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]]] - - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - - ##plt.plot(y_padded_up_to_down_padded) - ##plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') - ##plt.show() - - ##plt.plot(y_padded_up_to_down_padded) - ##plt.plot(peaks_neg_new,y_padded_up_to_down_padded[peaks_neg_new],'*') - ##plt.show() - - ##plt.plot(y_padded_smoothed) - ##plt.plot(peaks,y_padded_smoothed[peaks],'*') - ##plt.show() - - ##plt.plot(y_padded_smoothed) - ##plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') - ##plt.show() - - peaks=peaks_new_tot[:] - peaks_neg=peaks_neg_new[:] - - - else: - peaks_new_tot=peaks[:] - peaks=peaks_new_tot[:] - peaks_neg=peaks_neg_new[:] + coord_text.set("points", points_co) except: pass - - - mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks=np.std(y_padded_smoothed[peaks]) - peaks_values=y_padded_smoothed[peaks] - - peaks_neg = peaks_neg - 20 - 20 - peaks = peaks - 20 + print(dir_of_image) + print(self.f_name) + print(os.path.join(dir_of_image, self.f_name) + ".xml") + tree = ET.ElementTree(data) + tree.write(os.path.join(dir_of_image, self.f_name) + ".xml") - for jj in range(len(peaks_neg)): - if peaks_neg[jj] > len(x) - 1: - peaks_neg[jj] = len(x) - 1 + def write_into_page_xml(self, contours, page_coord, dir_of_image, order_of_texts, id_of_texts, all_found_texline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, curved_line, slopes, slopes_marginals): - for jj in range(len(peaks)): - if peaks[jj] > len(x) - 1: - peaks[jj] = len(x) - 1 - - + found_polygons_text_region = contours + ##found_polygons_text_region_h=contours_h - textline_boxes = [] - textline_boxes_rot = [] - - if len(peaks_neg) == len(peaks) + 1 and len(peaks) >= 3: - for jj in range(len(peaks)): - - if jj==(len(peaks)-1): - dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) - dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) - - if peaks_values[jj]>mean_value_of_peaks-std_value_of_peaks/2.: - point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down =y_max_cont-1##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) - else: - point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down =y_max_cont-1##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + # create the file structure + data = ET.Element("PcGts") - point_down_narrow = peaks[jj] + first_nonzero + int( - 1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./2) - else: - dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) - dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) - - if peaks_values[jj]>mean_value_of_peaks-std_value_of_peaks/2.: - point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) - else: - point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + data.set("xmlns", "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15") + data.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") + data.set("xsi:schemaLocation", "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15") - point_down_narrow = peaks[jj] + first_nonzero + int( - 1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + metadata = ET.SubElement(data, "Metadata") + author = ET.SubElement(metadata, "Creator") + author.text = "SBB_QURATOR" + created = ET.SubElement(metadata, "Created") + created.text = "2019-06-17T18:15:12" - if point_down_narrow >= img_patch.shape[0]: - point_down_narrow = img_patch.shape[0] - 2 + changetime = ET.SubElement(metadata, "LastChange") + changetime.text = "2019-06-17T18:15:12" - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) + page = ET.SubElement(data, "Page") - xvinside = xv[distances >= 0] + page.set("imageFilename", self.image_dir) + page.set("imageHeight", str(self.height_org)) + page.set("imageWidth", str(self.width_org)) + page.set("type", "content") + page.set("readingDirection", "left-to-right") + page.set("textLineOrder", "top-to-bottom") - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) - x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) - - p1 = np.dot(rotation_matrix, [int(x_min), int(point_up)]) - p2 = np.dot(rotation_matrix, [int(x_max), int(point_up)]) - p3 = np.dot(rotation_matrix, [int(x_max), int(point_down)]) - p4 = np.dot(rotation_matrix, [int(x_min), int(point_down)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - - - x_min_rot1=x_min_rot1-x_help - x_max_rot2=x_max_rot2-x_help - x_max_rot3=x_max_rot3-x_help - x_min_rot4=x_min_rot4-x_help - - point_up_rot1=point_up_rot1-y_help - point_up_rot2=point_up_rot2-y_help - point_down_rot3=point_down_rot3-y_help - point_down_rot4=point_down_rot4-y_help + page_print_sub = ET.SubElement(page, "PrintSpace") + coord_page = ET.SubElement(page_print_sub, "Coords") + points_page_print = "" - + for lmm in range(len(self.cont_page[0])): + if len(self.cont_page[0][lmm]) == 2: + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0]) / self.scale_x)) + points_page_print = points_page_print + "," + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][1]) / self.scale_y)) + else: + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0][0]) / self.scale_x)) + points_page_print = points_page_print + "," + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0][1]) / self.scale_y)) + if lmm < (len(self.cont_page[0]) - 1): + points_page_print = points_page_print + " " + coord_page.set("points", points_page_print) - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) + if len(contours) > 0: + region_order = ET.SubElement(page, "ReadingOrder") + region_order_sub = ET.SubElement(region_order, "OrderedGroup") - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) + region_order_sub.set("id", "ro357564684568544579089") - elif len(peaks) < 1: - pass + indexer_region = 0 - elif len(peaks) == 1: - - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[0] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) + for vj in order_of_texts: + name = "coord_text_" + str(vj) + name = ET.SubElement(region_order_sub, "RegionRefIndexed") - xvinside = xv[distances >= 0] - - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) - x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) - #x_min = x_min_cont - #x_max = x_max_cont - - y_min = y_min_cont - y_max = y_max_cont - - p1 = np.dot(rotation_matrix, [int(x_min), int(y_min)]) - p2 = np.dot(rotation_matrix, [int(x_max), int(y_min)]) - p3 = np.dot(rotation_matrix, [int(x_max), int(y_max)]) - p4 = np.dot(rotation_matrix, [int(x_min), int(y_max)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - - x_min_rot1=x_min_rot1-x_help - x_max_rot2=x_max_rot2-x_help - x_max_rot3=x_max_rot3-x_help - x_min_rot4=x_min_rot4-x_help - - point_up_rot1=point_up_rot1-y_help - point_up_rot2=point_up_rot2-y_help - point_down_rot3=point_down_rot3-y_help - point_down_rot4=point_down_rot4-y_help - - + name.set("index", str(indexer_region)) + name.set("regionRef", id_of_texts[vj]) + indexer_region += 1 + id_of_marginalia = [] + for vm in range(len(found_polygons_marginals)): + id_of_marginalia.append("r" + str(indexer_region)) - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) + name = "coord_text_" + str(indexer_region) + name = ET.SubElement(region_order_sub, "RegionRefIndexed") + name.set("index", str(indexer_region)) + name.set("regionRef", "r" + str(indexer_region)) + indexer_region += 1 - textline_boxes.append(np.array([[int(x_min), int(y_min)], - [int(x_max), int(y_min)], - [int(x_max), int(y_max)], - [int(x_min), int(y_max)]])) + id_indexer = 0 + id_indexer_l = 0 + for mm in range(len(found_polygons_text_region)): + textregion = ET.SubElement(page, "TextRegion") + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 - elif len(peaks) == 2: - dis_to_next = np.abs(peaks[1] - peaks[0]) - for jj in range(len(peaks)): - if jj == 0: - point_up = 0#peaks[jj] + first_nonzero - int(1. / 1.7 * dis_to_next) - if point_up < 0: - point_up = 1 - point_down = peaks[jj] + first_nonzero + int(1. / 1.8 * dis_to_next) - elif jj == 1: - point_down = peaks[jj] + first_nonzero + int(1. / 1.8 * dis_to_next) - if point_down >= img_patch.shape[0]: - point_down = img_patch.shape[0] - 2 - point_up = peaks[jj] + first_nonzero - int(1. / 1.8 * dis_to_next) + textregion.set("type", "paragraph") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) + points_co = "" + for lmm in range(len(found_polygons_text_region[mm])): + if len(found_polygons_text_region[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - xvinside = xv[distances >= 0] + if lmm < (len(found_polygons_text_region[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) - x_max = np.max(xvinside) - - p1 = np.dot(rotation_matrix, [int(x_min), int(point_up)]) - p2 = np.dot(rotation_matrix, [int(x_max), int(point_up)]) - p3 = np.dot(rotation_matrix, [int(x_max), int(point_down)]) - p4 = np.dot(rotation_matrix, [int(x_min), int(point_down)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - + for j in range(len(all_found_texline_polygons[mm])): - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - x_min_rot1=x_min_rot1-x_help - x_max_rot2=x_max_rot2-x_help - x_max_rot3=x_max_rot3-x_help - x_min_rot4=x_min_rot4-x_help - - point_up_rot1=point_up_rot1-y_help - point_up_rot2=point_up_rot2-y_help - point_down_rot3=point_down_rot3-y_help - point_down_rot4=point_down_rot4-y_help + textline = ET.SubElement(textregion, "TextLine") + textline.set("id", "l" + str(id_indexer_l)) - - - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) - else: - for jj in range(len(peaks)): - - if jj == 0: - dis_to_next = peaks[jj + 1] - peaks[jj] - # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) - point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next) - if point_up < 0: - point_up = 1 - # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) - point_down = peaks[jj] + first_nonzero + int(1. / 1.9 * dis_to_next) - elif jj == len(peaks) - 1: - dis_to_next = peaks[jj] - peaks[jj - 1] - # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) - point_down = peaks[jj] + first_nonzero + int(1. / 1.7 * dis_to_next) - if point_down >= img_patch.shape[0]: - point_down = img_patch.shape[0] - 2 - # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) - point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next) - else: - dis_to_next_down = peaks[jj + 1] - peaks[jj] - dis_to_next_up = peaks[jj] - peaks[jj - 1] + id_indexer_l += 1 - point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next_up) - point_down = peaks[jj] + first_nonzero + int(1. / 1.9 * dis_to_next_down) + coord = ET.SubElement(textline, "Coords") - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) + texteq = ET.SubElement(textline, "TextEquiv") - xvinside = xv[distances >= 0] + uni = ET.SubElement(texteq, "Unicode") + uni.text = " " - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) - x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) - - p1 = np.dot(rotation_matrix, [int(x_min), int(point_up)]) - p2 = np.dot(rotation_matrix, [int(x_max), int(point_up)]) - p3 = np.dot(rotation_matrix, [int(x_max), int(point_down)]) - p4 = np.dot(rotation_matrix, [int(x_min), int(point_down)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - - x_min_rot1=x_min_rot1-x_help - x_max_rot2=x_max_rot2-x_help - x_max_rot3=x_max_rot3-x_help - x_min_rot4=x_min_rot4-x_help - - point_up_rot1=point_up_rot1-y_help - point_up_rot2=point_up_rot2-y_help - point_down_rot3=point_down_rot3-y_help - point_down_rot4=point_down_rot4-y_help - + # points = ET.SubElement(coord, 'Points') + points_co = "" + for l in range(len(all_found_texline_polygons[mm][j])): + # point = ET.SubElement(coord, 'Point') + + if not self.curved_line: + # point.set('x',str(found_polygons[j][l][0])) + # point.set('y',str(found_polygons[j][l][1])) + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + + if (self.curved_line) and abs(slopes[mm]) <= 45: + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + page_coord[0]) / self.scale_y)) + + elif (self.curved_line) and abs(slopes[mm]) > 45: + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + all_box_coord[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + all_box_coord[mm][0] + page_coord[0]) / self.scale_y)) + if l < (len(all_found_texline_polygons[mm][j]) - 1): + points_co = points_co + " " + # print(points_co) + coord.set("points", points_co) - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) + texteqreg = ET.SubElement(textregion, "TextEquiv") - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) + unireg = ET.SubElement(texteqreg, "Unicode") + unireg.text = " " - - return peaks, textline_boxes_rot - - def return_rotated_contours(self,slope,img_patch): - dst = self.rotate_image(img_patch, slope) - dst = dst.astype(np.uint8) - dst = dst[:, :, 0] - dst[dst != 0] = 1 - - imgray = cv2.cvtColor(dst, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel) - thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel) - contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - return contours - - def seperate_lines_vertical_cont(self, img_patch, contour_text_interest, thetha,box_ind,add_boxes_coor_into_textlines): - kernel = np.ones((5, 5), np.uint8) - pixel=255 - min_area=0 - max_area=1 - - - if len(img_patch.shape)==3: - cnts_images=(img_patch[:,:,0]==pixel)*1 - else: - cnts_images=(img_patch[:,:]==pixel)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=max_area,min_area=min_area) - - - cont_final=[] - ###print(add_boxes_coor_into_textlines,'ikki') - for i in range(len(contours_imgs)): - img_contour = np.zeros((cnts_images.shape[0], cnts_images.shape[1], 3)) - img_contour = cv2.fillPoly(img_contour, pts=[contours_imgs[i]], color=(255, 255, 255)) - - - - img_contour = img_contour.astype(np.uint8) - - img_contour = cv2.dilate(img_contour, kernel, iterations=4) - imgrayrot = cv2.cvtColor(img_contour, cv2.COLOR_BGR2GRAY) - _, threshrot = cv2.threshold(imgrayrot, 0, 255, 0) - contours_text_rot, _ = cv2.findContours(threshrot.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - ##contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[ - ##0] - ##contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] - ##if add_boxes_coor_into_textlines: - ##print(np.shape(contours_text_rot[0]),'sjppo') - ##contours_text_rot[0][:, 0, 0]=contours_text_rot[0][:, 0, 0] + box_ind[0] - ##contours_text_rot[0][:, 0, 1]=contours_text_rot[0][:, 0, 1] + box_ind[1] - cont_final.append(contours_text_rot[0]) - - ##print(cont_final,'nadizzzz') - return None,cont_final - - def seperate_lines_vertical(self, img_patch, contour_text_interest, thetha): - - - thetha=thetha+90 - - (h, w) = img_patch.shape[:2] - center = (w // 2, h // 2) - M = cv2.getRotationMatrix2D(center, -thetha, 1.0) - x_d = M[0, 2] - y_d = M[1, 2] - - thetha = thetha / 180. * np.pi - rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) - contour_text_interest_copy = contour_text_interest.copy() - - x_cont = contour_text_interest[:, 0, 0] - y_cont = contour_text_interest[:, 0, 1] - x_cont = x_cont - np.min(x_cont) - y_cont = y_cont - np.min(y_cont) - - x_min_cont = 0 - x_max_cont = img_patch.shape[1] - y_min_cont = 0 - y_max_cont = img_patch.shape[0] - - xv = np.linspace(x_min_cont, x_max_cont, 1000) - - textline_patch_sum_along_width = img_patch.sum(axis=0) - - first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) - - y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] - y_padded = np.zeros(len(y) + 40) - y_padded[20:len(y) + 20] = y - x = np.array(range(len(y))) - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - if 1>0: + ###print(len(contours_h)) + ###if len(contours_h)>0: + ###for mm in range(len(found_polygons_text_region_h)): + ###textregion=ET.SubElement(page, 'TextRegion') + ###try: + ###id_indexer=id_indexer + ###id_indexer_l=id_indexer_l + ###except: + ###id_indexer=0 + ###id_indexer_l=0 + ###textregion.set('id','r'+str(id_indexer)) + ###id_indexer+=1 + + ###textregion.set('type','header') + ####if mm==0: + #### textregion.set('type','header') + ####else: + #### textregion.set('type','paragraph') + ###coord_text = ET.SubElement(textregion, 'Coords') + + ###points_co='' + ###for lmm in range(len(found_polygons_text_region_h[mm])): + + ###if len(found_polygons_text_region_h[mm][lmm])==2: + ###points_co=points_co+str( int( (found_polygons_text_region_h[mm][lmm][0] +page_coord[2])/self.scale_x ) ) + ###points_co=points_co+',' + ###points_co=points_co+str( int( (found_polygons_text_region_h[mm][lmm][1] +page_coord[0])/self.scale_y ) ) + ###else: + ###points_co=points_co+str( int((found_polygons_text_region_h[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) + ###points_co=points_co+',' + ###points_co=points_co+str( int((found_polygons_text_region_h[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) + + ###if lmm<(len(found_polygons_text_region_h[mm])-1): + ###points_co=points_co+' ' + ####print(points_co) + ###coord_text.set('points',points_co) + + ###for j in range(len(all_found_texline_polygons_h[mm])): + + ###textline=ET.SubElement(textregion, 'TextLine') + + ###textline.set('id','l'+str(id_indexer_l)) + + ###id_indexer_l+=1 + + ###coord = ET.SubElement(textline, 'Coords') + + ###texteq=ET.SubElement(textline, 'TextEquiv') + + ###uni=ET.SubElement(texteq, 'Unicode') + ###uni.text = ' ' + + ####points = ET.SubElement(coord, 'Points') + + ###points_co='' + ###for l in range(len(all_found_texline_polygons_h[mm][j])): + ####point = ET.SubElement(coord, 'Point') + + ####point.set('x',str(found_polygons[j][l][0])) + ####point.set('y',str(found_polygons[j][l][1])) + ###if len(all_found_texline_polygons_h[mm][j][l])==2: + ###points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] +page_coord[2] + ###+all_box_coord_h[mm][2])/self.scale_x) ) + ###points_co=points_co+',' + ###points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] +page_coord[0] + ###+all_box_coord_h[mm][0])/self.scale_y) ) + ###else: + ###points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] +page_coord[2] + ###+all_box_coord_h[mm][2])/self.scale_x ) ) + ###points_co=points_co+',' + ###points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] +page_coord[0] + ###+all_box_coord_h[mm][0])/self.scale_y) ) + + ###if l<(len(all_found_texline_polygons_h[mm][j])-1): + ###points_co=points_co+' ' + ####print(points_co) + ###coord.set('points',points_co) + + ###texteqreg=ET.SubElement(textregion, 'TextEquiv') + + ###unireg=ET.SubElement(texteqreg, 'Unicode') + ###unireg.text = ' ' + try: + # id_indexer_l=0 try: - - y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) - y_padded_up_to_down_e=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) - y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e - y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) - - - peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) - peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) - neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - - arg_neg_must_be_deleted= np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3 ] - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - - - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - - - peaks_new=peaks_e[:] - peaks_neg_new=peaks_neg_e[:] - - clusters_to_be_deleted=[] - if len(arg_diff_cluster)>0: - - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - - - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] - - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - - - else: - peaks_new_tot=peaks_e[:] - - - textline_con,hierachy=self.return_contours_of_image(img_patch) - textline_con_fil=self.filter_contours_area_of_image(img_patch,textline_con,hierachy,max_area=1,min_area=0.0008) - y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) - - sigma_gaus=int( y_diff_mean * (7./40.0) ) - #print(sigma_gaus,'sigma_gaus') + ###id_indexer=id_indexer + id_indexer_l = id_indexer_l except: - sigma_gaus=12 - if sigma_gaus<3: - sigma_gaus=3 - #print(sigma_gaus,'sigma') - - - y_padded_smoothed= gaussian_filter1d(y_padded, sigma_gaus) - y_padded_up_to_down=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded=np.zeros(len(y_padded_up_to_down)+40) - y_padded_up_to_down_padded[20:len(y_padded_up_to_down)+20]=y_padded_up_to_down - y_padded_up_to_down_padded= gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) - - - peaks, _ = find_peaks(y_padded_smoothed, height=0) - peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) - - - #plt.plot(y_padded_up_to_down_padded) - #plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') - #plt.title('negs') - #plt.show() - - - - #plt.plot(y_padded_smoothed) - #plt.plot(peaks,y_padded_smoothed[peaks],'*') - #plt.title('poss') - #plt.show() - - - - neg_peaks_max=np.max(y_padded_up_to_down_padded[peaks_neg]) - - - arg_neg_must_be_deleted= np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.42 ] + ###id_indexer=0 + id_indexer_l = 0 + for mm in range(len(found_polygons_marginals)): + textregion = ET.SubElement(page, "TextRegion") - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - + textregion.set("id", id_of_marginalia[mm]) - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - + textregion.set("type", "marginalia") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") - peaks_new=peaks[:] - peaks_neg_new=peaks_neg[:] - clusters_to_be_deleted=[] - + points_co = "" + for lmm in range(len(found_polygons_marginals[mm])): + if len(found_polygons_marginals[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_marginals[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) - if len(arg_diff_cluster)>=2 and len(arg_diff_cluster)>0: - - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - elif len(arg_neg_must_be_deleted)>=2 and len(arg_diff_cluster)==0: - clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - - - - if len(arg_neg_must_be_deleted)==1: - clusters_to_be_deleted.append(arg_neg_must_be_deleted) - + if lmm < (len(found_polygons_marginals[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]]] - - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - + for j in range(len(all_found_texline_polygons_marginals[mm])): - peaks=peaks_new_tot[:] - peaks_neg=peaks_neg_new[:] - - - else: - peaks_new_tot=peaks[:] - peaks=peaks_new_tot[:] - peaks_neg=peaks_neg_new[:] - - mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks=np.std(y_padded_smoothed[peaks]) - peaks_values=y_padded_smoothed[peaks] - + textline = ET.SubElement(textregion, "TextLine") - peaks_neg = peaks_neg - 20 - 20 - peaks = peaks - 20 + textline.set("id", "l" + str(id_indexer_l)) - for jj in range(len(peaks_neg)): - if peaks_neg[jj] > len(x) - 1: - peaks_neg[jj] = len(x) - 1 + id_indexer_l += 1 - for jj in range(len(peaks)): - if peaks[jj] > len(x) - 1: - peaks[jj] = len(x) - 1 - - + coord = ET.SubElement(textline, "Coords") - textline_boxes = [] - textline_boxes_rot = [] + texteq = ET.SubElement(textline, "TextEquiv") - if len(peaks_neg) == len(peaks) + 1 and len(peaks) >= 3: - #print('11') - for jj in range(len(peaks)): - - if jj==(len(peaks)-1): - dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) - dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) - - if peaks_values[jj]>mean_value_of_peaks-std_value_of_peaks/2.: - point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down =x_max_cont-1##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) - else: - point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down =x_max_cont-1##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + uni = ET.SubElement(texteq, "Unicode") + uni.text = " " - point_down_narrow = peaks[jj] + first_nonzero + int( - 1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./2) - else: - dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) - dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) - - if peaks_values[jj]>mean_value_of_peaks-std_value_of_peaks/2.: - point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) - else: - point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) - point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + # points = ET.SubElement(coord, 'Points') - point_down_narrow = peaks[jj] + first_nonzero + int( - 1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + points_co = "" + for l in range(len(all_found_texline_polygons_marginals[mm][j])): + # point = ET.SubElement(coord, 'Point') + + if not self.curved_line: + # point.set('x',str(found_polygons[j][l][0])) + # point.set('y',str(found_polygons[j][l][1])) + if len(all_found_texline_polygons_marginals[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0] + all_box_coord_marginals[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][1] + all_box_coord_marginals[mm][0] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][0] + all_box_coord_marginals[mm][2] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][1] + all_box_coord_marginals[mm][0] + page_coord[0]) / self.scale_y)) + + if self.curved_line: + if len(all_found_texline_polygons_marginals[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons_marginals[mm][j][l][0][1] + page_coord[0]) / self.scale_y)) + + if l < (len(all_found_texline_polygons_marginals[mm][j]) - 1): + points_co = points_co + " " + # print(points_co) + coord.set("points", points_co) + except: + pass + try: + for mm in range(len(found_polygons_text_region_img)): + textregion = ET.SubElement(page, "ImageRegion") - if point_down_narrow >= img_patch.shape[0]: - point_down_narrow = img_patch.shape[0] - 2 + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) + coord_text = ET.SubElement(textregion, "Coords") + points_co = "" + for lmm in range(len(found_polygons_text_region_img[mm])): + points_co = points_co + str(int((found_polygons_text_region_img[mm][lmm, 0, 0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region_img[mm][lmm, 0, 1] + page_coord[0]) / self.scale_y)) - xvinside = xv[distances >= 0] + if lmm < (len(found_polygons_text_region_img[mm]) - 1): + points_co = points_co + " " - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) - x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) - - p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) - p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) - p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) - p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) - - elif len(peaks) < 1: + coord_text.set("points", points_co) + ###for mm in range(len(found_polygons_text_region_img)): + ###textregion=ET.SubElement(page, 'ImageRegion') + + ###textregion.set('id','r'+str(id_indexer)) + ###id_indexer+=1 + + ###coord_text = ET.SubElement(textregion, 'Coords') + ###print(found_polygons_text_region_img[mm]) + ###points_co='' + ###for lmm in range(len(found_polygons_text_region_img[mm])): + ###print(len(found_polygons_text_region_img[mm][lmm])) + + ###if len(found_polygons_text_region_img[mm][lmm])==2: + ###points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm][0]+page_coord[2] )/self.scale_x ) ) + ###points_co=points_co+',' + ###points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm][1]+page_coord[0] )/self.scale_y ) ) + ###else: + ###points_co=points_co+str( int((found_polygons_text_region_img[mm][lmm][0][0]+page_coord[2] )/self.scale_x) ) + ###points_co=points_co+',' + ###points_co=points_co+str( int((found_polygons_text_region_img[mm][lmm][0][1]+page_coord[0] )/self.scale_y) ) + + ###if lmm<(len(found_polygons_text_region_img[mm])-1): + ###points_co=points_co+' ' + + ###coord_text.set('points',points_co) + except: pass - elif len(peaks) == 1: - x_min = x_min_cont - x_max = x_max_cont - - y_min = y_min_cont - y_max = y_max_cont - - p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) - p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) - p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) - p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - - textline_boxes.append(np.array([[int(x_min), int(y_min)], - [int(x_max), int(y_min)], - [int(x_max), int(y_max)], - [int(x_min), int(y_max)]])) - - - - elif len(peaks) == 2: - dis_to_next = np.abs(peaks[1] - peaks[0]) - for jj in range(len(peaks)): - if jj == 0: - point_up = 0#peaks[jj] + first_nonzero - int(1. / 1.7 * dis_to_next) - if point_up < 0: - point_up = 1 - point_down = peaks[jj] + first_nonzero + int(1. / 1.8 * dis_to_next) - elif jj == 1: - point_down = peaks[jj] + first_nonzero + int(1. / 1.8 * dis_to_next) - if point_down >= img_patch.shape[0]: - point_down = img_patch.shape[0] - 2 - point_up = peaks[jj] + first_nonzero - int(1. / 1.8 * dis_to_next) - - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) - - xvinside = xv[distances >= 0] - - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) - x_max = np.max(xvinside) - - p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) - p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) - p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) - p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) - else: - for jj in range(len(peaks)): - - if jj == 0: - dis_to_next = peaks[jj + 1] - peaks[jj] - # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) - point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next) - if point_up < 0: - point_up = 1 - # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) - point_down = peaks[jj] + first_nonzero + int(1. / 1.9 * dis_to_next) - elif jj == len(peaks) - 1: - dis_to_next = peaks[jj] - peaks[jj - 1] - # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) - point_down = peaks[jj] + first_nonzero + int(1. / 1.7 * dis_to_next) - if point_down >= img_patch.shape[0]: - point_down = img_patch.shape[0] - 2 - # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) - point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next) - else: - dis_to_next_down = peaks[jj + 1] - peaks[jj] - dis_to_next_up = peaks[jj] - peaks[jj - 1] - - point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next_up) - point_down = peaks[jj] + first_nonzero + int(1. / 1.9 * dis_to_next_down) - - distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) - for mj in range(len(xv))] - distances = np.array(distances) + ####try: + ####for mm in range(len(found_polygons_tables)): + ####textregion=ET.SubElement(page, 'TableRegion') - xvinside = xv[distances >= 0] + ####textregion.set('id','r'+str(id_indexer)) + ####id_indexer+=1 - if len(xvinside) == 0: - x_min = x_min_cont - x_max = x_max_cont - else: - x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) - x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) - - p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) - p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) - p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) - p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) - - x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d - x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d - x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d - x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d - - - if x_min_rot1<0: - x_min_rot1=0 - if x_min_rot4<0: - x_min_rot4=0 - if point_up_rot1<0: - point_up_rot1=0 - if point_up_rot2<0: - point_up_rot2=0 - + ####coord_text = ET.SubElement(textregion, 'Coords') + ####points_co='' + ####for lmm in range(len(found_polygons_tables[mm])): - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) + ####if len(found_polygons_tables[mm][lmm])==2: + ####points_co=points_co+str( int( (found_polygons_tables[mm][lmm][0] +page_coord[2])/self.scale_x ) ) + ####points_co=points_co+',' + ####points_co=points_co+str( int( (found_polygons_tables[mm][lmm][1] +page_coord[0])/self.scale_y ) ) + ####else: + ####points_co=points_co+str( int((found_polygons_tables[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) + ####points_co=points_co+',' + ####points_co=points_co+str( int((found_polygons_tables[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) + ####if lmm<(len(found_polygons_tables[mm])-1): + ####points_co=points_co+' ' + ####coord_text.set('points',points_co) + ####except: + ####pass + """ - return peaks, textline_boxes_rot - - def textline_contours_postprocessing(self, textline_mask, slope, contour_text_interest, box_ind,slope_first,add_boxes_coor_into_textlines=False): - - textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 - textline_mask = textline_mask.astype(np.uint8) - kernel = np.ones((5, 5), np.uint8) - textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_OPEN, kernel) - textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, kernel) - textline_mask = cv2.erode(textline_mask, kernel, iterations=2) - #textline_mask = cv2.erode(textline_mask, kernel, iterations=1) - - #print(textline_mask.shape[0]/float(textline_mask.shape[1]),'miz') try: - #if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - #plt.imshow(textline_mask) - #plt.show() - - #if abs(slope)>1: - #x_help=30 - #y_help=2 - #else: - #x_help=2 - #y_help=2 - - x_help=30 - y_help=2 - - - textline_mask_help=np.zeros((textline_mask.shape[0]+int(2*y_help),textline_mask.shape[1]+int(2*x_help),3)) - textline_mask_help[y_help:y_help+textline_mask.shape[0],x_help:x_help+textline_mask.shape[1],:]=np.copy(textline_mask[:,:,:]) - - - - dst = self.rotate_image(textline_mask_help, slope) - dst = dst[:, :, 0] - dst[dst != 0] = 1 - - #if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - #plt.imshow(dst) - #plt.show() - - contour_text_copy = contour_text_interest.copy() - - contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[ - 0] - contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] - - img_contour = np.zeros((box_ind[3], box_ind[2], 3)) - img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=(255, 255, 255)) - - #if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - #plt.imshow(img_contour) - #plt.show() - - img_contour_help=np.zeros((img_contour.shape[0]+int(2*y_help),img_contour.shape[1]+int(2*x_help),3)) - + for mm in range(len(found_polygons_drop_capitals)): + textregion=ET.SubElement(page, 'DropCapitals') - img_contour_help[y_help:y_help+img_contour.shape[0],x_help:x_help+img_contour.shape[1],:]=np.copy(img_contour[:,:,:]) - + textregion.set('id','r'+str(id_indexer)) + id_indexer+=1 - - img_contour_rot = self.rotate_image(img_contour_help, slope) - - - - - #plt.imshow(img_contour_rot_help) - #plt.show() - - #plt.imshow(dst_help) - #plt.show() - - #if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: - #plt.imshow(img_contour_rot_help) - #plt.show() - - #plt.imshow(dst_help) - #plt.show() + coord_text = ET.SubElement(textregion, 'Coords') - img_contour_rot = img_contour_rot.astype(np.uint8) - #dst_help = dst_help.astype(np.uint8) - imgrayrot = cv2.cvtColor(img_contour_rot, cv2.COLOR_BGR2GRAY) - _, threshrot = cv2.threshold(imgrayrot, 0, 255, 0) - contours_text_rot, _ = cv2.findContours(threshrot.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + points_co='' + for lmm in range(len(found_polygons_drop_capitals[mm])): - len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] - ind_big_con = np.argmax(len_con_text_rot) - - #print('juzaa') - if abs(slope)>45: - #print(add_boxes_coor_into_textlines,'avval') - _, contours_rotated_clean = self.seperate_lines_vertical_cont(textline_mask, contours_text_rot[ind_big_con], box_ind,slope,add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) - else: - _, contours_rotated_clean = self.seperate_lines(dst, contours_text_rot[ind_big_con], slope,x_help,y_help) + if len(found_polygons_drop_capitals[mm][lmm])==2: + points_co=points_co+str( int( (found_polygons_drop_capitals[mm][lmm][0] +page_coord[2])/self.scale_x ) ) + points_co=points_co+',' + points_co=points_co+str( int( (found_polygons_drop_capitals[mm][lmm][1] +page_coord[0])/self.scale_y ) ) + else: + points_co=points_co+str( int((found_polygons_drop_capitals[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) + points_co=points_co+',' + points_co=points_co+str( int((found_polygons_drop_capitals[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) + if lmm<(len(found_polygons_drop_capitals[mm])-1): + points_co=points_co+' ' + coord_text.set('points',points_co) except: + pass + """ - contours_rotated_clean = [] - - return contours_rotated_clean - - def textline_contours_to_get_slope_correctly(self, textline_mask, img_patch, contour_interest): - - slope_new = 0 # deskew_images(img_patch) + # print(dir_of_image) + print(self.f_name) + # print(os.path.join(dir_of_image, self.f_name) + ".xml") + tree = ET.ElementTree(data) + tree.write(os.path.join(dir_of_image, self.f_name) + ".xml") + # cv2.imwrite(os.path.join(dir_of_image, self.f_name) + ".tif",self.image_org) - textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 + def get_regions_from_xy_2models(self,img,is_image_enhanced): + img_org=np.copy(img) - textline_mask = textline_mask.astype(np.uint8) - textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_OPEN, self.kernel) - textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, self.kernel) - textline_mask = cv2.erode(textline_mask, self.kernel, iterations=1) - imgray = cv2.cvtColor(textline_mask, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) + img_height_h=img_org.shape[0] + img_width_h=img_org.shape[1] - thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, self.kernel) - thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, self.kernel) + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) - contours, hirarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + gaussian_filter=False + patches=True + binary=False - main_contours = self.filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area=1, min_area=0.003) - textline_maskt = textline_mask[:, :, 0] - textline_maskt[textline_maskt != 0] = 1 - peaks_point, _ = self.seperate_lines(textline_maskt, contour_interest, slope_new) - mean_dis = np.mean(np.diff(peaks_point)) - len_x = thresh.shape[1] + ratio_y=1.3 + ratio_x=1 - slope_lines = [] - contours_slope_new = [] + median_blur=False - for kk in range(len(main_contours)): + img= resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - if len(main_contours[kk].shape)==2: - xminh=np.min(main_contours[kk][:,0]) - xmaxh=np.max(main_contours[kk][:,0]) + if binary: + img = otsu_copy_binary(img)#self.otsu_copy(img) + img = img.astype(np.uint16) - yminh=np.min(main_contours[kk][:,1]) - ymaxh=np.max(main_contours[kk][:,1]) - elif len(main_contours[kk].shape)==3: - xminh=np.min(main_contours[kk][:,0,0]) - xmaxh=np.max(main_contours[kk][:,0,0]) + if median_blur: + img=cv2.medianBlur(img,5) + if gaussian_filter: + img= cv2.GaussianBlur(img,(5,5),0) + img = img.astype(np.uint16) + prediction_regions_org_y=self.do_prediction(patches,img,model_region) - yminh=np.min(main_contours[kk][:,0,1]) - ymaxh=np.max(main_contours[kk][:,0,1]) + prediction_regions_org_y = resize_image(prediction_regions_org_y, img_height_h, img_width_h ) + #plt.imshow(prediction_regions_org_y[:,:,0]) + #plt.show() + #sys.exit() + prediction_regions_org_y=prediction_regions_org_y[:,:,0] - if ymaxh - yminh <= mean_dis and ( - xmaxh - xminh) >= 0.3 * len_x: # xminh>=0.05*len_x and xminh<=0.4*len_x and xmaxh<=0.95*len_x and xmaxh>=0.6*len_x: - contours_slope_new.append(main_contours[kk]) - rows, cols = thresh.shape[:2] - [vx, vy, x, y] = cv2.fitLine(main_contours[kk], cv2.DIST_L2, 0, 0.01, 0.01) + mask_zeros_y=(prediction_regions_org_y[:,:]==0)*1 - slope_lines.append((vy / vx) / np.pi * 180) - if len(slope_lines) >= 2: - slope = np.mean(slope_lines) # slope_true/np.pi*180 - else: - slope = 999 - else: - slope = 0 - return slope - def return_contours_of_image(self,image): - - if len(image.shape)==2: - image=np.repeat(image[:, :, np.newaxis], 3, axis=2) - image=image.astype(np.uint8) + if is_image_enhanced: + ratio_x=1.2 else: - image=image.astype(np.uint8) - imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - return contours,hierachy - - def find_contours_mean_y_diff(self,contours_main): - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - return np.mean( np.diff( np.sort( np.array(cy_main) ) ) ) - - - def isNaN(self,num): - return num != num - - def find_num_col_olddd(self,regions_without_seperators,sigma_,multiplier=3.8 ): - regions_without_seperators_0=regions_without_seperators[:,:].sum(axis=1) - - meda_n_updown=regions_without_seperators_0[len(regions_without_seperators_0)::-1] - - first_nonzero=(next((i for i, x in enumerate(regions_without_seperators_0) if x), 0)) - last_nonzero=(next((i for i, x in enumerate(meda_n_updown) if x), 0)) - - last_nonzero=len(regions_without_seperators_0)-last_nonzero - - - y=regions_without_seperators_0#[first_nonzero:last_nonzero] - - y_help=np.zeros(len(y)+20) - - y_help[10:len(y)+10]=y - - x=np.array( range(len(y)) ) - - - - - zneg_rev=-y_help+np.max(y_help) - - zneg=np.zeros(len(zneg_rev)+20) - - zneg[10:len(zneg_rev)+10]=zneg_rev - - z=gaussian_filter1d(y, sigma_) - zneg= gaussian_filter1d(zneg, sigma_) - - - peaks_neg, _ = find_peaks(zneg, height=0) - peaks, _ = find_peaks(z, height=0) - - peaks_neg=peaks_neg-10-10 - - - - last_nonzero=last_nonzero-0#100 - first_nonzero=first_nonzero+0#+100 - - peaks_neg=peaks_neg[(peaks_neg>first_nonzero) & (peaks_neg.06*regions_without_seperators.shape[1]) & (peaks<0.94*regions_without_seperators.shape[1])] - - interest_pos=z[peaks] - - interest_pos=interest_pos[interest_pos>10] - - interest_neg=z[peaks_neg] - - - if interest_neg[0]<0.1: - interest_neg=interest_neg[1:] - if interest_neg[len(interest_neg)-1]<0.1: - interest_neg=interest_neg[:len(interest_neg)-1] - - - - min_peaks_pos=np.min(interest_pos) - min_peaks_neg=0#np.min(interest_neg) - - - dis_talaei=(min_peaks_pos-min_peaks_neg)/multiplier - grenze=min_peaks_pos-dis_talaei#np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 - - interest_neg_fin=interest_neg#[(interest_negfirst_nonzero) & (peaks_neg.06*regions_without_seperators.shape[1]) & (peaks<0.94*regions_without_seperators.shape[1])] - ###""" - ###interest_pos=z[peaks] - - ###interest_pos=interest_pos[interest_pos>10] - - ###interest_neg=z[peaks_neg] - - ###min_peaks_pos=np.mean(interest_pos) - ###min_peaks_neg=0#np.min(interest_neg) - - ###dis_talaei=(min_peaks_pos-min_peaks_neg)/multiplier - ####print(interest_pos) - ###grenze=min_peaks_pos-dis_talaei#np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 - - ###interest_neg_fin=interest_neg[(interest_negimg_patch_org.shape[0]: - - #plt.imshow(img_resized) - #plt.show() - angels=np.array([-45, 0 , 45 , 90 , ])#np.linspace(-12,12,100)#np.array([0 , 45 , 90 , -45]) - - #res=[] - #num_of_peaks=[] - #index_cor=[] - var_res=[] - - #indexer=0 - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - #plt.imshow(img_rot) - #plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - - - #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - #print(var_spectrum,'var_spectrum') - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - ##print(rot,var_spectrum,'var_spectrum') - #res_me=np.mean(neg_peaks) - #if res_me==0: - #res_me=1000000000000000000000 - #else: - #pass - - #res_num=len(neg_peaks) - except: - #res_me=1000000000000000000000 - #res_num=0 - var_spectrum=0 - #if self.isNaN(res_me): - #pass - #else: - #res.append( res_me ) - #var_res.append(var_spectrum) - #num_of_peaks.append( res_num ) - #index_cor.append(indexer) - #indexer=indexer+1 - - var_res.append(var_spectrum) - #index_cor.append(indexer) - #indexer=indexer+1 - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - - angels=np.linspace(ang_int-22.5,ang_int+22.5,100) - - #res=[] - #num_of_peaks=[] - #index_cor=[] - var_res=[] - - - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - ##plt.imshow(img_rot) - ##plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - elif main_page and img_patch_org.shape[1]<=img_patch_org.shape[0]: - - #plt.imshow(img_resized) - #plt.show() - angels=np.linspace(-12,12,100)#np.array([0 , 45 , 90 , -45]) - - - var_res=[] - - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - #plt.imshow(img_rot) - #plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - - - #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - #print(var_spectrum,'var_spectrum') - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - if self.dir_of_all is not None: - #print('galdi?') - plt.figure(figsize=(60,30)) - plt.rcParams['font.size']='50' - plt.plot(angels,np.array(var_res),'-o',markersize=25,linewidth=4) - plt.xlabel('angle',fontsize=50) - plt.ylabel('variance of sum of rotated textline in direction of x axis',fontsize=50) - - plt.plot(angels[np.argmax(var_res)],var_res[np.argmax(np.array(var_res))] ,'*',markersize=50,label='Angle of deskewing=' +str("{:.2f}".format(angels[np.argmax(var_res)]))+r'$\degree$') - plt.legend(loc='best') - plt.savefig(os.path.join(self.dir_of_all,self.f_name+'_rotation_angle.png')) - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - - early_slope_edge=11 - if abs(ang_int)>early_slope_edge and ang_int<0: - - angels=np.linspace(-90,-12,100) - - var_res=[] - - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - ##plt.imshow(img_rot) - ##plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - elif abs(ang_int)>early_slope_edge and ang_int>0: - - angels=np.linspace(90,12,100) - - - var_res=[] - - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - ##plt.imshow(img_rot) - ##plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - #print(indexer,'indexer') - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - else: - - - angels=np.linspace(-25,25,60) - - var_res=[] - - indexer=0 - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - #plt.imshow(img_rot) - #plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - - - #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - #print(var_spectrum,'var_spectrum') - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - #plt.plot(var_res) - #plt.show() - - ##plt.plot(mom3_res) - ##plt.show() - #print(ang_int,'ang_int111') - - early_slope_edge=22 - if abs(ang_int)>early_slope_edge and ang_int<0: - - angels=np.linspace(-90,-25,60) - - var_res=[] - - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - ##plt.imshow(img_rot) - ##plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - elif abs(ang_int)>early_slope_edge and ang_int>0: - - angels=np.linspace(90,25,60) - - var_res=[] - - indexer=0 - for rot in angels: - img_rot=self.rotate_image(img_resized,rot) - ##plt.imshow(img_rot) - ##plt.show() - img_rot[img_rot!=0]=1 - #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) - try: - var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) - #print(indexer,'indexer') - except: - var_spectrum=0 - - var_res.append(var_spectrum) - - - - try: - var_res=np.array(var_res) - - ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - return ang_int - - def return_deskew_slope_new(self,img_patch,sigma_des): - max_x_y=max(img_patch.shape[0],img_patch.shape[1]) - - ##img_patch=self.resize_image(img_patch,max_x_y,max_x_y) - - - - img_patch_copy=np.zeros((img_patch.shape[0],img_patch.shape[1])) - img_patch_copy[:,:]=img_patch[:,:]#img_patch_org[:,:,0] - - - img_patch_padded=np.zeros((int( max_x_y*(1.4) ) , int( max_x_y*(1.4) ) )) - - img_patch_padded_center_p=int(img_patch_padded.shape[0]/2.) - len_x_org_patch_half=int(img_patch_copy.shape[1]/2.) - len_y_org_patch_half=int(img_patch_copy.shape[0]/2.) - - img_patch_padded[img_patch_padded_center_p-len_y_org_patch_half:img_patch_padded_center_p-len_y_org_patch_half+img_patch_copy.shape[0],img_patch_padded_center_p-len_x_org_patch_half:img_patch_padded_center_p-len_x_org_patch_half+img_patch_copy.shape[1] ]=img_patch_copy[:,:] - #img_patch_padded[ int( img_patch_copy.shape[0]*(.1)):int( img_patch_copy.shape[0]*(.1))+img_patch_copy.shape[0] , int( img_patch_copy.shape[1]*(.8)):int( img_patch_copy.shape[1]*(.8))+img_patch_copy.shape[1] ]=img_patch_copy[:,:] - angles=np.linspace(-25,25,80) - - res=[] - num_of_peaks=[] - index_cor=[] - var_res=[] - - #plt.imshow(img_patch) - #plt.show() - indexer=0 - for rot in angles: - #print(rot,'rot') - img_rotated=self.rotate_image(img_patch_padded,rot) - img_rotated[img_rotated!=0]=1 - - #plt.imshow(img_rotated) - #plt.show() - - try: - neg_peaks,var_spectrum=self.get_standard_deviation_of_summed_textline_patch_along_width(img_rotated,sigma_des,20.3 ) - res_me=np.mean(neg_peaks) - if res_me==0: - res_me=1000000000000000000000 - else: - pass - - res_num=len(neg_peaks) - except: - res_me=1000000000000000000000 - res_num=0 - var_spectrum=0 - if self.isNaN(res_me): - pass - else: - res.append( res_me ) - var_res.append(var_spectrum) - num_of_peaks.append( res_num ) - index_cor.append(indexer) - indexer=indexer+1 - - - try: - var_res=np.array(var_res) - #print(var_res) - - ang_int=angles[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - if abs(ang_int)>15: - angles=np.linspace(-90,-50,30) - res=[] - num_of_peaks=[] - index_cor=[] - var_res=[] - - #plt.imshow(img_patch) - #plt.show() - indexer=0 - for rot in angles: - #print(rot,'rot') - img_rotated=self.rotate_image(img_patch_padded,rot) - img_rotated[img_rotated!=0]=1 - - #plt.imshow(img_rotated) - #plt.show() - - try: - neg_peaks,var_spectrum=self.get_standard_deviation_of_summed_textline_patch_along_width(img_rotated,sigma_des,20.3 ) - res_me=np.mean(neg_peaks) - if res_me==0: - res_me=1000000000000000000000 - else: - pass - - res_num=len(neg_peaks) - except: - res_me=1000000000000000000000 - res_num=0 - var_spectrum=0 - if self.isNaN(res_me): - pass - else: - res.append( res_me ) - var_res.append(var_spectrum) - num_of_peaks.append( res_num ) - index_cor.append(indexer) - indexer=indexer+1 - - - try: - var_res=np.array(var_res) - #print(var_res) - - ang_int=angles[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] - except: - ang_int=0 - - return ang_int - - - def do_work_of_slopes(self,q,poly,box_sub,boxes_per_process,textline_mask_tot,contours_per_process): - slope_biggest=0 - slopes_sub = [] - boxes_sub_new=[] - poly_sub=[] - for mv in range(len(boxes_per_process)): - - - crop_img, _ = self.crop_image_inside_box(boxes_per_process[mv], - np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) - crop_img=crop_img[:,:,0] - crop_img=cv2.erode(crop_img,self.kernel,iterations = 2) - - try: - textline_con,hierachy=self.return_contours_of_image(crop_img) - textline_con_fil=self.filter_contours_area_of_image(crop_img,textline_con,hierachy,max_area=1,min_area=0.0008) - y_diff_mean=self.find_contours_mean_y_diff(textline_con_fil) - - sigma_des=int( y_diff_mean * (4./40.0) ) - - if sigma_des<1: - sigma_des=1 - - crop_img[crop_img>0]=1 - slope_corresponding_textregion=self.return_deskew_slop(crop_img,sigma_des) - - - except: - slope_corresponding_textregion=999 - - if slope_corresponding_textregion==999: - slope_corresponding_textregion=slope_biggest - ##if np.abs(slope_corresponding_textregion)>12.5 and slope_corresponding_textregion!=999: - ##slope_corresponding_textregion=slope_biggest - ##elif slope_corresponding_textregion==999: - ##slope_corresponding_textregion=slope_biggest - slopes_sub.append(slope_corresponding_textregion) - - cnt_clean_rot = self.textline_contours_postprocessing(crop_img - , slope_corresponding_textregion, - contours_per_process[mv], boxes_per_process[mv]) - - poly_sub.append(cnt_clean_rot) - boxes_sub_new.append(boxes_per_process[mv] ) - - - q.put(slopes_sub) - poly.put(poly_sub) - box_sub.put(boxes_sub_new ) - - def get_slopes_and_deskew(self, contours,textline_mask_tot): - - slope_biggest=0#self.return_deskew_slop(img_int_p,sigma_des) - - num_cores = cpu_count() - q = Queue() - poly=Queue() - box_sub=Queue() - - processes = [] - nh=np.linspace(0, len(self.boxes), num_cores+1) - - - for i in range(num_cores): - boxes_per_process=self.boxes[int(nh[i]):int(nh[i+1])] - contours_per_process=contours[int(nh[i]):int(nh[i+1])] - processes.append(Process(target=self.do_work_of_slopes, args=(q,poly,box_sub, boxes_per_process, textline_mask_tot, contours_per_process))) - - for i in range(num_cores): - processes[i].start() - - self.slopes = [] - self.all_found_texline_polygons=[] - self.boxes=[] - - for i in range(num_cores): - slopes_for_sub_process=q.get(True) - boxes_for_sub_process=box_sub.get(True) - polys_for_sub_process=poly.get(True) - - for j in range(len(slopes_for_sub_process)): - self.slopes.append(slopes_for_sub_process[j]) - self.all_found_texline_polygons.append(polys_for_sub_process[j]) - self.boxes.append(boxes_for_sub_process[j]) - - for i in range(num_cores): - processes[i].join() - - - def order_of_regions_old(self, textline_mask,contours_main): - mada_n=textline_mask.sum(axis=1) - y=mada_n[:] - - y_help=np.zeros(len(y)+40) - y_help[20:len(y)+20]=y - x=np.array( range(len(y)) ) - - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - - - sigma_gaus=8 - - z= gaussian_filter1d(y_help, sigma_gaus) - zneg_rev=-y_help+np.max(y_help) - - zneg=np.zeros(len(zneg_rev)+40) - zneg[20:len(zneg_rev)+20]=zneg_rev - zneg= gaussian_filter1d(zneg, sigma_gaus) - - - peaks, _ = find_peaks(z, height=0) - peaks_neg, _ = find_peaks(zneg, height=0) - - peaks_neg=peaks_neg-20-20 - peaks=peaks-20 - - - - if contours_main!=None: - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - - - - if contours_main!=None: - indexer_main=np.array(range(len(contours_main))) - - - if contours_main!=None: - len_main=len(contours_main) - else: - len_main=0 - - - matrix_of_orders=np.zeros((len_main,5)) - - matrix_of_orders[:,0]=np.array( range( len_main ) ) - - matrix_of_orders[:len_main,1]=1 - matrix_of_orders[len_main:,1]=2 - - matrix_of_orders[:len_main,2]=cx_main - matrix_of_orders[:len_main,3]=cy_main - - matrix_of_orders[:len_main,4]=np.array( range( len_main ) ) - - peaks_neg_new=[] - peaks_neg_new.append(0) - for iii in range(len(peaks_neg)): - peaks_neg_new.append(peaks_neg[iii]) - peaks_neg_new.append(textline_mask.shape[0]) - - final_indexers_sorted=[] - for i in range(len(peaks_neg_new)-1): - top=peaks_neg_new[i] - down=peaks_neg_new[i+1] - - indexes_in=matrix_of_orders[:,0][(matrix_of_orders[:,3]>=top) & ((matrix_of_orders[:,3]=top) & ((matrix_of_orders[:,3]0: - - id_indexer=0 - id_indexer_l=0 - - for mm in range(len(found_polygons_text_region)): - textregion=ET.SubElement(page, 'TextRegion') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - textregion.set('type','paragraph') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_text_region[mm])): - if len(found_polygons_text_region[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_text_region[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_text_region[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_text_region[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_text_region[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_text_region[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - - - for j in range(len(all_found_texline_polygons[mm])): - - textline=ET.SubElement(textregion, 'TextLine') - - textline.set('id','l'+str(id_indexer_l)) - - id_indexer_l+=1 - - - coord = ET.SubElement(textline, 'Coords') - - texteq=ET.SubElement(textline, 'TextEquiv') - - uni=ET.SubElement(texteq, 'Unicode') - uni.text = ' ' - - #points = ET.SubElement(coord, 'Points') - - points_co='' - for l in range(len(all_found_texline_polygons[mm][j])): - #point = ET.SubElement(coord, 'Point') - - - - #point.set('x',str(found_polygons[j][l][0])) - #point.set('y',str(found_polygons[j][l][1])) - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] +page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] +page_coord[0])/self.scale_y) ) - - if l<(len(all_found_texline_polygons[mm][j])-1): - points_co=points_co+' ' - #print(points_co) - coord.set('points',points_co) - - texteqreg=ET.SubElement(textregion, 'TextEquiv') - - unireg=ET.SubElement(texteqreg, 'Unicode') - unireg.text = ' ' - - - - - - #print(dir_of_image) - print(self.f_name) - #print(os.path.join(dir_of_image, self.f_name) + ".xml") - tree = ET.ElementTree(data) - tree.write(os.path.join(dir_of_image, self.f_name) + ".xml") - - - - def write_into_page_xml_full(self,contours,contours_h,page_coord,dir_of_image,order_of_texts , id_of_texts,all_found_texline_polygons,all_found_texline_polygons_h,all_box_coord,all_box_coord_h,found_polygons_text_region_img,found_polygons_tables,found_polygons_drop_capitals,found_polygons_marginals,all_found_texline_polygons_marginals,all_box_coord_marginals,slopes,slopes_marginals): - - found_polygons_text_region=contours - found_polygons_text_region_h=contours_h - - - # create the file structure - data = ET.Element('PcGts') - - data.set('xmlns',"http://schema.primaresearch.org/PAGE/gts/pagecontent/2017-07-15") - data.set('xmlns:xsi',"http://www.w3.org/2001/XMLSchema-instance") - data.set('xsi:schemaLocation',"http://schema.primaresearch.org/PAGE/gts/pagecontent/2017-07-15") - - - - metadata=ET.SubElement(data,'Metadata') - - author=ET.SubElement(metadata, 'Creator') - author.text = 'SBB_QURATOR' - - - created=ET.SubElement(metadata, 'Created') - created.text = '2019-06-17T18:15:12' - - changetime=ET.SubElement(metadata, 'LastChange') - changetime.text = '2019-06-17T18:15:12' - - - - page=ET.SubElement(data,'Page') - - page.set('imageFilename', self.image_dir) - page.set('imageHeight',str(self.height_org) ) - page.set('imageWidth',str(self.width_org) ) - page.set('type',"content") - page.set('readingDirection',"left-to-right") - page.set('textLineOrder',"top-to-bottom" ) - - - - page_print_sub=ET.SubElement(page, 'PrintSpace') - coord_page = ET.SubElement(page_print_sub, 'Coords') - points_page_print='' - - for lmm in range(len(self.cont_page[0])): - if len(self.cont_page[0][lmm])==2: - points_page_print=points_page_print+str( int( (self.cont_page[0][lmm][0])/self.scale_x ) ) - points_page_print=points_page_print+',' - points_page_print=points_page_print+str( int( (self.cont_page[0][lmm][1])/self.scale_y ) ) - else: - points_page_print=points_page_print+str( int((self.cont_page[0][lmm][0][0])/self.scale_x) ) - points_page_print=points_page_print+',' - points_page_print=points_page_print+str( int((self.cont_page[0][lmm][0][1])/self.scale_y) ) - - if lmm<(len(self.cont_page[0])-1): - points_page_print=points_page_print+' ' - coord_page.set('points',points_page_print) - - - - - if len(contours)>0: - region_order=ET.SubElement(page, 'ReadingOrder') - region_order_sub = ET.SubElement(region_order, 'OrderedGroup') - - region_order_sub.set('id',"ro357564684568544579089") - - #args_sort=order_of_texts - for vj in order_of_texts: - name="coord_text_"+str(vj) - name = ET.SubElement(region_order_sub, 'RegionRefIndexed') - name.set('index',str(order_of_texts[vj]) ) - name.set('regionRef',id_of_texts[vj]) - - id_of_marginalia=[] - indexer_region=len(contours)+len(contours_h) - for vm in range(len(found_polygons_marginals)): - id_of_marginalia.append('r'+str(indexer_region)) - - name="coord_text_"+str(indexer_region) - name = ET.SubElement(region_order_sub, 'RegionRefIndexed') - name.set('index',str(indexer_region) ) - name.set('regionRef','r'+str(indexer_region)) - indexer_region+=1 - - - id_indexer=0 - id_indexer_l=0 - - for mm in range(len(found_polygons_text_region)): - textregion=ET.SubElement(page, 'TextRegion') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - textregion.set('type','paragraph') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_text_region[mm])): - if len(found_polygons_text_region[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_text_region[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_text_region[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_text_region[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_text_region[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_text_region[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - - - - - for j in range(len(all_found_texline_polygons[mm])): - - textline=ET.SubElement(textregion, 'TextLine') - - textline.set('id','l'+str(id_indexer_l)) - - id_indexer_l+=1 - - - coord = ET.SubElement(textline, 'Coords') - - texteq=ET.SubElement(textline, 'TextEquiv') - - uni=ET.SubElement(texteq, 'Unicode') - uni.text = ' ' - - #points = ET.SubElement(coord, 'Points') - - points_co='' - for l in range(len(all_found_texline_polygons[mm][j])): - #point = ET.SubElement(coord, 'Point') - - - if curved_line is None or curved_line=='false' or curved_line=='False' or curved_line=='FALSE' : - #point.set('x',str(found_polygons[j][l][0])) - #point.set('y',str(found_polygons[j][l][1])) - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - - if (curved_line=='true' or curved_line=='True' or curved_line=='TRUE') and np.abs(slopes[mm])<=45 : - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] - +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] - +page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] - +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] - +page_coord[0])/self.scale_y) ) - elif (curved_line=='true' or curved_line=='True' or curved_line=='TRUE') and np.abs(slopes[mm])>45 : - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - - if l<(len(all_found_texline_polygons[mm][j])-1): - points_co=points_co+' ' - #print(points_co) - coord.set('points',points_co) - - texteqreg=ET.SubElement(textregion, 'TextEquiv') - - unireg=ET.SubElement(texteqreg, 'Unicode') - unireg.text = ' ' - - - print(len(contours_h)) - if len(contours_h)>0: - for mm in range(len(found_polygons_text_region_h)): - textregion=ET.SubElement(page, 'TextRegion') - try: - id_indexer=id_indexer - id_indexer_l=id_indexer_l - except: - id_indexer=0 - id_indexer_l=0 - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - textregion.set('type','header') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_text_region_h[mm])): - - if len(found_polygons_text_region_h[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_text_region_h[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_text_region_h[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_text_region_h[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_text_region_h[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_text_region_h[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - - for j in range(len(all_found_texline_polygons_h[mm])): - - textline=ET.SubElement(textregion, 'TextLine') - - textline.set('id','l'+str(id_indexer_l)) - - id_indexer_l+=1 - - - coord = ET.SubElement(textline, 'Coords') - - texteq=ET.SubElement(textline, 'TextEquiv') - - uni=ET.SubElement(texteq, 'Unicode') - uni.text = ' ' - - #points = ET.SubElement(coord, 'Points') - - points_co='' - for l in range(len(all_found_texline_polygons_h[mm][j])): - #point = ET.SubElement(coord, 'Point') - - - if curved_line is None or curved_line=='false' or curved_line=='False' or curved_line=='FALSE' : - #point.set('x',str(found_polygons[j][l][0])) - #point.set('y',str(found_polygons[j][l][1])) - if len(all_found_texline_polygons_h[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] - +all_box_coord_h[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] - +all_box_coord_h[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] - +all_box_coord_h[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] - +all_box_coord_h[mm][0]+page_coord[0])/self.scale_y) ) - - if curved_line=='true' or curved_line=='True' or curved_line=='TRUE' : - if len(all_found_texline_polygons_h[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] - +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] - +page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] - +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] - +page_coord[0])/self.scale_y) ) - - if l<(len(all_found_texline_polygons_h[mm][j])-1): - points_co=points_co+' ' - #print(points_co) - coord.set('points',points_co) - - texteqreg=ET.SubElement(textregion, 'TextEquiv') - - unireg=ET.SubElement(texteqreg, 'Unicode') - unireg.text = ' ' - - - - - - - if len(found_polygons_drop_capitals)>0: - id_indexer=len(contours_h)+len(contours)+len(found_polygons_marginals) - for mm in range(len(found_polygons_drop_capitals)): - textregion=ET.SubElement(page, 'TextRegion') - - - #id_indexer_l=id_indexer_l - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - textregion.set('type','drop-capital') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_drop_capitals[mm])): - - if len(found_polygons_drop_capitals[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_drop_capitals[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_drop_capitals[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_drop_capitals[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_drop_capitals[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_drop_capitals[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - - ##for j in range(len(all_found_texline_polygons_h[mm])): - - ##textline=ET.SubElement(textregion, 'TextLine') - - ##textline.set('id','l'+str(id_indexer_l)) - - ##id_indexer_l+=1 - - - ##coord = ET.SubElement(textline, 'Coords') - - ##texteq=ET.SubElement(textline, 'TextEquiv') - - ##uni=ET.SubElement(texteq, 'Unicode') - ##uni.text = ' ' - - ###points = ET.SubElement(coord, 'Points') - - ##points_co='' - ##for l in range(len(all_found_texline_polygons_h[mm][j])): - ###point = ET.SubElement(coord, 'Point') - - - ##if curved_line is None or curved_line=='false' or curved_line=='False' or curved_line=='FALSE' : - ###point.set('x',str(found_polygons[j][l][0])) - ###point.set('y',str(found_polygons[j][l][1])) - ##if len(all_found_texline_polygons_h[mm][j][l])==2: - ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] - ##+all_box_coord_h[mm][2]+page_coord[2])/self.scale_x) ) - ##points_co=points_co+',' - ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] - ##+all_box_coord_h[mm][0]+page_coord[0])/self.scale_y) ) - ##else: - ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] - ##+all_box_coord_h[mm][2]+page_coord[2])/self.scale_x ) ) - ##points_co=points_co+',' - ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] - ##+all_box_coord_h[mm][0]+page_coord[0])/self.scale_y) ) - - ##if curved_line=='true' or curved_line=='True' or curved_line=='TRUE' : - ##if len(all_found_texline_polygons_h[mm][j][l])==2: - ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] - ##+page_coord[2])/self.scale_x) ) - ##points_co=points_co+',' - ##points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] - ##+page_coord[0])/self.scale_y) ) - ##else: - ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] - ##+page_coord[2])/self.scale_x ) ) - ##points_co=points_co+',' - ##points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] - ##+page_coord[0])/self.scale_y) ) - - ##if l<(len(all_found_texline_polygons_h[mm][j])-1): - ##points_co=points_co+' ' - ###print(points_co) - ####coord.set('points',points_co) - - texteqreg=ET.SubElement(textregion, 'TextEquiv') - - unireg=ET.SubElement(texteqreg, 'Unicode') - unireg.text = ' ' - - - - - - try: - - try: - ###id_indexer=id_indexer - id_indexer_l=id_indexer_l - except: - ###id_indexer=0 - id_indexer_l=0 - for mm in range(len(found_polygons_marginals)): - textregion=ET.SubElement(page, 'TextRegion') - - textregion.set('id',id_of_marginalia[mm]) - - textregion.set('type','marginalia') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_marginals[mm])): - if len(found_polygons_marginals[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_marginals[mm][lmm][0]+page_coord[2] )/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_marginals[mm][lmm][1]+page_coord[0] )/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_marginals[mm][lmm][0][0]+page_coord[2] )/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_marginals[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_marginals[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - - for j in range(len(all_found_texline_polygons_marginals[mm])): - - textline=ET.SubElement(textregion, 'TextLine') - - textline.set('id','l'+str(id_indexer_l)) - - id_indexer_l+=1 - - - coord = ET.SubElement(textline, 'Coords') - - texteq=ET.SubElement(textline, 'TextEquiv') - - uni=ET.SubElement(texteq, 'Unicode') - uni.text = ' ' - - #points = ET.SubElement(coord, 'Points') - - points_co='' - for l in range(len(all_found_texline_polygons_marginals[mm][j])): - #point = ET.SubElement(coord, 'Point') - - - if curved_line is None or curved_line=='false' or curved_line=='False' or curved_line=='FALSE' : - #point.set('x',str(found_polygons[j][l][0])) - #point.set('y',str(found_polygons[j][l][1])) - if len(all_found_texline_polygons_marginals[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][0] - +all_box_coord_marginals[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][1] - +all_box_coord_marginals[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][0] - +all_box_coord_marginals[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][1] - +all_box_coord_marginals[mm][0]+page_coord[0])/self.scale_y) ) - - if curved_line=='true' or curved_line=='True' or curved_line=='TRUE' : - if len(all_found_texline_polygons_marginals[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][0] - +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][1] - +page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][0] - +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][1] - +page_coord[0])/self.scale_y) ) - - if l<(len(all_found_texline_polygons_marginals[mm][j])-1): - points_co=points_co+' ' - #print(points_co) - coord.set('points',points_co) - - - texteqreg=ET.SubElement(textregion, 'TextEquiv') - - unireg=ET.SubElement(texteqreg, 'Unicode') - unireg.text = ' ' - except: - pass - - try: - id_indexer=len(contours_h)+len(contours)+len(found_polygons_marginals)+len(found_polygons_drop_capitals) - for mm in range(len(found_polygons_text_region_img)): - textregion=ET.SubElement(page, 'ImageRegion') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_text_region_img[mm])): - - if len(found_polygons_text_region_img[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_text_region_img[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_text_region_img[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_text_region_img[mm])-1): - points_co=points_co+' ' - - - coord_text.set('points',points_co) - except: - pass - - - try: - for mm in range(len(found_polygons_tables)): - textregion=ET.SubElement(page, 'TableRegion') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_tables[mm])): - - if len(found_polygons_tables[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_tables[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_tables[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_tables[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_tables[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_tables[mm])-1): - points_co=points_co+' ' - - - coord_text.set('points',points_co) - except: - pass - - - - - print(dir_of_image) - print(self.f_name) - print(os.path.join(dir_of_image, self.f_name) + ".xml") - tree = ET.ElementTree(data) - tree.write(os.path.join(dir_of_image, self.f_name) + ".xml") - - def write_into_page_xml(self,contours,page_coord,dir_of_image,order_of_texts , id_of_texts,all_found_texline_polygons,all_box_coord,found_polygons_text_region_img,found_polygons_marginals,all_found_texline_polygons_marginals,all_box_coord_marginals,curved_line,slopes,slopes_marginals): - - found_polygons_text_region=contours - ##found_polygons_text_region_h=contours_h - - - # create the file structure - data = ET.Element('PcGts') - - data.set('xmlns',"http://schema.primaresearch.org/PAGE/gts/pagecontent/2017-07-15") - data.set('xmlns:xsi',"http://www.w3.org/2001/XMLSchema-instance") - data.set('xsi:schemaLocation',"http://schema.primaresearch.org/PAGE/gts/pagecontent/2017-07-15") - - - - metadata=ET.SubElement(data,'Metadata') - - author=ET.SubElement(metadata, 'Creator') - author.text = 'SBB_QURATOR' - - - created=ET.SubElement(metadata, 'Created') - created.text = '2019-06-17T18:15:12' - - changetime=ET.SubElement(metadata, 'LastChange') - changetime.text = '2019-06-17T18:15:12' - - - - page=ET.SubElement(data,'Page') - - page.set('imageFilename', self.image_dir) - page.set('imageHeight',str(self.height_org) ) - page.set('imageWidth',str(self.width_org) ) - page.set('type',"content") - page.set('readingDirection',"left-to-right") - page.set('textLineOrder',"top-to-bottom" ) - - - - page_print_sub=ET.SubElement(page, 'PrintSpace') - coord_page = ET.SubElement(page_print_sub, 'Coords') - points_page_print='' - - for lmm in range(len(self.cont_page[0])): - if len(self.cont_page[0][lmm])==2: - points_page_print=points_page_print+str( int( (self.cont_page[0][lmm][0])/self.scale_x ) ) - points_page_print=points_page_print+',' - points_page_print=points_page_print+str( int( (self.cont_page[0][lmm][1])/self.scale_y ) ) - else: - points_page_print=points_page_print+str( int((self.cont_page[0][lmm][0][0])/self.scale_x) ) - points_page_print=points_page_print+',' - points_page_print=points_page_print+str( int((self.cont_page[0][lmm][0][1])/self.scale_y) ) - - if lmm<(len(self.cont_page[0])-1): - points_page_print=points_page_print+' ' - coord_page.set('points',points_page_print) - - - if len(contours)>0: - region_order=ET.SubElement(page, 'ReadingOrder') - region_order_sub = ET.SubElement(region_order, 'OrderedGroup') - - region_order_sub.set('id',"ro357564684568544579089") - - indexer_region=0 - - - for vj in order_of_texts: - name="coord_text_"+str(vj) - name = ET.SubElement(region_order_sub, 'RegionRefIndexed') - - name.set('index',str(indexer_region) ) - name.set('regionRef',id_of_texts[vj]) - indexer_region+=1 - - id_of_marginalia=[] - for vm in range(len(found_polygons_marginals)): - id_of_marginalia.append('r'+str(indexer_region)) - - name="coord_text_"+str(indexer_region) - name = ET.SubElement(region_order_sub, 'RegionRefIndexed') - name.set('index',str(indexer_region) ) - name.set('regionRef','r'+str(indexer_region)) - indexer_region+=1 - - - - - - id_indexer=0 - id_indexer_l=0 - - for mm in range(len(found_polygons_text_region)): - textregion=ET.SubElement(page, 'TextRegion') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - textregion.set('type','paragraph') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_text_region[mm])): - if len(found_polygons_text_region[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_text_region[mm][lmm][0]+page_coord[2] )/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_text_region[mm][lmm][1]+page_coord[0] )/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_text_region[mm][lmm][0][0]+page_coord[2] )/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_text_region[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_text_region[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - - - for j in range(len(all_found_texline_polygons[mm])): - - textline=ET.SubElement(textregion, 'TextLine') - - textline.set('id','l'+str(id_indexer_l)) - - id_indexer_l+=1 - - - coord = ET.SubElement(textline, 'Coords') - - texteq=ET.SubElement(textline, 'TextEquiv') - - uni=ET.SubElement(texteq, 'Unicode') - uni.text = ' ' - - #points = ET.SubElement(coord, 'Points') - - points_co='' - for l in range(len(all_found_texline_polygons[mm][j])): - #point = ET.SubElement(coord, 'Point') - - - if curved_line is None or curved_line=='false' or curved_line=='False' or curved_line=='FALSE' : - #point.set('x',str(found_polygons[j][l][0])) - #point.set('y',str(found_polygons[j][l][1])) - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - - if (curved_line=='true' or curved_line=='True' or curved_line=='TRUE') and abs(slopes[mm])<=45 : - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] - +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] - +page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] - +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] - +page_coord[0])/self.scale_y) ) - - elif (curved_line=='true' or curved_line=='True' or curved_line=='TRUE') and abs(slopes[mm])>45: - if len(all_found_texline_polygons[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons[mm][j][l][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][0] - +all_box_coord[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons[mm][j][l][0][1] - +all_box_coord[mm][0]+page_coord[0])/self.scale_y) ) - - - if l<(len(all_found_texline_polygons[mm][j])-1): - points_co=points_co+' ' - #print(points_co) - coord.set('points',points_co) - - texteqreg=ET.SubElement(textregion, 'TextEquiv') - - unireg=ET.SubElement(texteqreg, 'Unicode') - unireg.text = ' ' - - ###print(len(contours_h)) - ###if len(contours_h)>0: - ###for mm in range(len(found_polygons_text_region_h)): - ###textregion=ET.SubElement(page, 'TextRegion') - ###try: - ###id_indexer=id_indexer - ###id_indexer_l=id_indexer_l - ###except: - ###id_indexer=0 - ###id_indexer_l=0 - ###textregion.set('id','r'+str(id_indexer)) - ###id_indexer+=1 - - ###textregion.set('type','header') - ####if mm==0: - #### textregion.set('type','header') - ####else: - #### textregion.set('type','paragraph') - ###coord_text = ET.SubElement(textregion, 'Coords') - - ###points_co='' - ###for lmm in range(len(found_polygons_text_region_h[mm])): - - ###if len(found_polygons_text_region_h[mm][lmm])==2: - ###points_co=points_co+str( int( (found_polygons_text_region_h[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - ###points_co=points_co+',' - ###points_co=points_co+str( int( (found_polygons_text_region_h[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - ###else: - ###points_co=points_co+str( int((found_polygons_text_region_h[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - ###points_co=points_co+',' - ###points_co=points_co+str( int((found_polygons_text_region_h[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - ###if lmm<(len(found_polygons_text_region_h[mm])-1): - ###points_co=points_co+' ' - ####print(points_co) - ###coord_text.set('points',points_co) - - ###for j in range(len(all_found_texline_polygons_h[mm])): - - ###textline=ET.SubElement(textregion, 'TextLine') - - ###textline.set('id','l'+str(id_indexer_l)) - - ###id_indexer_l+=1 - - - ###coord = ET.SubElement(textline, 'Coords') - - ###texteq=ET.SubElement(textline, 'TextEquiv') - - ###uni=ET.SubElement(texteq, 'Unicode') - ###uni.text = ' ' - - ####points = ET.SubElement(coord, 'Points') - - ###points_co='' - ###for l in range(len(all_found_texline_polygons_h[mm][j])): - ####point = ET.SubElement(coord, 'Point') - - - - ####point.set('x',str(found_polygons[j][l][0])) - ####point.set('y',str(found_polygons[j][l][1])) - ###if len(all_found_texline_polygons_h[mm][j][l])==2: - ###points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][0] +page_coord[2] - ###+all_box_coord_h[mm][2])/self.scale_x) ) - ###points_co=points_co+',' - ###points_co=points_co+str( int( (all_found_texline_polygons_h[mm][j][l][1] +page_coord[0] - ###+all_box_coord_h[mm][0])/self.scale_y) ) - ###else: - ###points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][0] +page_coord[2] - ###+all_box_coord_h[mm][2])/self.scale_x ) ) - ###points_co=points_co+',' - ###points_co=points_co+str( int( ( all_found_texline_polygons_h[mm][j][l][0][1] +page_coord[0] - ###+all_box_coord_h[mm][0])/self.scale_y) ) - - ###if l<(len(all_found_texline_polygons_h[mm][j])-1): - ###points_co=points_co+' ' - ####print(points_co) - ###coord.set('points',points_co) - - ###texteqreg=ET.SubElement(textregion, 'TextEquiv') - - ###unireg=ET.SubElement(texteqreg, 'Unicode') - ###unireg.text = ' ' - try: - #id_indexer_l=0 - - try: - ###id_indexer=id_indexer - id_indexer_l=id_indexer_l - except: - ###id_indexer=0 - id_indexer_l=0 - - for mm in range(len(found_polygons_marginals)): - textregion=ET.SubElement(page, 'TextRegion') - - textregion.set('id',id_of_marginalia[mm]) - - textregion.set('type','marginalia') - #if mm==0: - # textregion.set('type','header') - #else: - # textregion.set('type','paragraph') - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_marginals[mm])): - if len(found_polygons_marginals[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_marginals[mm][lmm][0]+page_coord[2] )/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_marginals[mm][lmm][1]+page_coord[0] )/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_marginals[mm][lmm][0][0]+page_coord[2] )/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_marginals[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_marginals[mm])-1): - points_co=points_co+' ' - #print(points_co) - coord_text.set('points',points_co) - - for j in range(len(all_found_texline_polygons_marginals[mm])): - - textline=ET.SubElement(textregion, 'TextLine') - - textline.set('id','l'+str(id_indexer_l)) - - id_indexer_l+=1 - - - coord = ET.SubElement(textline, 'Coords') - - texteq=ET.SubElement(textline, 'TextEquiv') - - uni=ET.SubElement(texteq, 'Unicode') - uni.text = ' ' - - #points = ET.SubElement(coord, 'Points') - - points_co='' - for l in range(len(all_found_texline_polygons_marginals[mm][j])): - #point = ET.SubElement(coord, 'Point') - - - if curved_line is None or curved_line=='false' or curved_line=='False' or curved_line=='FALSE' : - #point.set('x',str(found_polygons[j][l][0])) - #point.set('y',str(found_polygons[j][l][1])) - if len(all_found_texline_polygons_marginals[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][0] - +all_box_coord_marginals[mm][2]+page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][1] - +all_box_coord_marginals[mm][0]+page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][0] - +all_box_coord_marginals[mm][2]+page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][1] - +all_box_coord_marginals[mm][0]+page_coord[0])/self.scale_y) ) - - if curved_line=='true' or curved_line=='True' or curved_line=='TRUE' : - if len(all_found_texline_polygons_marginals[mm][j][l])==2: - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][0] - +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int( (all_found_texline_polygons_marginals[mm][j][l][1] - +page_coord[0])/self.scale_y) ) - else: - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][0] - +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( ( all_found_texline_polygons_marginals[mm][j][l][0][1] - +page_coord[0])/self.scale_y) ) - - if l<(len(all_found_texline_polygons_marginals[mm][j])-1): - points_co=points_co+' ' - #print(points_co) - coord.set('points',points_co) - except: - pass - - try: - - for mm in range(len(found_polygons_text_region_img)): - textregion=ET.SubElement(page, 'ImageRegion') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - - coord_text = ET.SubElement(textregion, 'Coords') - points_co='' - for lmm in range(len(found_polygons_text_region_img[mm])): - points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm,0,0]+page_coord[2] )/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm,0,1]+page_coord[0] )/self.scale_y ) ) - - if lmm<(len(found_polygons_text_region_img[mm])-1): - points_co=points_co+' ' - - - coord_text.set('points',points_co) - ###for mm in range(len(found_polygons_text_region_img)): - ###textregion=ET.SubElement(page, 'ImageRegion') - - ###textregion.set('id','r'+str(id_indexer)) - ###id_indexer+=1 - - - ###coord_text = ET.SubElement(textregion, 'Coords') - ###print(found_polygons_text_region_img[mm]) - ###points_co='' - ###for lmm in range(len(found_polygons_text_region_img[mm])): - ###print(len(found_polygons_text_region_img[mm][lmm])) - - ###if len(found_polygons_text_region_img[mm][lmm])==2: - ###points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm][0]+page_coord[2] )/self.scale_x ) ) - ###points_co=points_co+',' - ###points_co=points_co+str( int( (found_polygons_text_region_img[mm][lmm][1]+page_coord[0] )/self.scale_y ) ) - ###else: - ###points_co=points_co+str( int((found_polygons_text_region_img[mm][lmm][0][0]+page_coord[2] )/self.scale_x) ) - ###points_co=points_co+',' - ###points_co=points_co+str( int((found_polygons_text_region_img[mm][lmm][0][1]+page_coord[0] )/self.scale_y) ) - - ###if lmm<(len(found_polygons_text_region_img[mm])-1): - ###points_co=points_co+' ' - - - ###coord_text.set('points',points_co) - except: - pass - - - ####try: - ####for mm in range(len(found_polygons_tables)): - ####textregion=ET.SubElement(page, 'TableRegion') - - ####textregion.set('id','r'+str(id_indexer)) - ####id_indexer+=1 - - - ####coord_text = ET.SubElement(textregion, 'Coords') - - ####points_co='' - ####for lmm in range(len(found_polygons_tables[mm])): - - ####if len(found_polygons_tables[mm][lmm])==2: - ####points_co=points_co+str( int( (found_polygons_tables[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - ####points_co=points_co+',' - ####points_co=points_co+str( int( (found_polygons_tables[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - ####else: - ####points_co=points_co+str( int((found_polygons_tables[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - ####points_co=points_co+',' - ####points_co=points_co+str( int((found_polygons_tables[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - ####if lmm<(len(found_polygons_tables[mm])-1): - ####points_co=points_co+' ' - - - ####coord_text.set('points',points_co) - ####except: - ####pass - """ - - try: - for mm in range(len(found_polygons_drop_capitals)): - textregion=ET.SubElement(page, 'DropCapitals') - - textregion.set('id','r'+str(id_indexer)) - id_indexer+=1 - - - coord_text = ET.SubElement(textregion, 'Coords') - - points_co='' - for lmm in range(len(found_polygons_drop_capitals[mm])): - - if len(found_polygons_drop_capitals[mm][lmm])==2: - points_co=points_co+str( int( (found_polygons_drop_capitals[mm][lmm][0] +page_coord[2])/self.scale_x ) ) - points_co=points_co+',' - points_co=points_co+str( int( (found_polygons_drop_capitals[mm][lmm][1] +page_coord[0])/self.scale_y ) ) - else: - points_co=points_co+str( int((found_polygons_drop_capitals[mm][lmm][0][0] +page_coord[2])/self.scale_x) ) - points_co=points_co+',' - points_co=points_co+str( int((found_polygons_drop_capitals[mm][lmm][0][1] +page_coord[0])/self.scale_y) ) - - if lmm<(len(found_polygons_drop_capitals[mm])-1): - points_co=points_co+' ' - - - coord_text.set('points',points_co) - except: - pass - """ - - - #print(dir_of_image) - print(self.f_name) - #print(os.path.join(dir_of_image, self.f_name) + ".xml") - tree = ET.ElementTree(data) - tree.write(os.path.join(dir_of_image, self.f_name) + ".xml") - #cv2.imwrite(os.path.join(dir_of_image, self.f_name) + ".tif",self.image_org) - - def deskew_region_prediction(self,regions_prediction, slope): - image_regions_deskewd=np.zeros(regions_prediction[:,:].shape) - for ind in np.unique(regions_prediction[:,:]): - interest_reg=(regions_prediction[:,:]==ind)*1 - interest_reg=interest_reg.astype(np.uint8) - deskewed_new=self.rotate_image(interest_reg,slope) - deskewed_new=deskewed_new[:,:] - deskewed_new[deskewed_new!=0]=ind - - image_regions_deskewd=image_regions_deskewd+deskewed_new - return image_regions_deskewd - - - def deskew_erarly(self,textline_mask): - textline_mask_org=np.copy(textline_mask) - #print(textline_mask.shape,np.unique(textline_mask),'hizzzzz') - #slope_new=0#deskew_images(img_patch) - - textline_mask=np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2)*255 - - textline_mask=textline_mask.astype(np.uint8) - kernel = np.ones((5,5),np.uint8) - - - imgray = cv2.cvtColor(textline_mask, cv2.COLOR_BGR2GRAY) - - - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - - contours,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - #print(hirarchy) - - - commenst_contours=self.filter_contours_area_of_image(thresh,contours,hirarchy,max_area=0.01,min_area=0.003) - main_contours=self.filter_contours_area_of_image(thresh,contours,hirarchy,max_area=1,min_area=0.003) - interior_contours=self.filter_contours_area_of_image_interiors(thresh,contours,hirarchy,max_area=1,min_area=0) - - img_comm=np.zeros(thresh.shape) - img_comm_in=cv2.fillPoly(img_comm, pts =main_contours, color=(255,255,255)) - ###img_comm_in=cv2.fillPoly(img_comm, pts =interior_contours, color=(0,0,0)) - - - img_comm_in=np.repeat(img_comm_in[:, :, np.newaxis], 3, axis=2) - img_comm_in=img_comm_in.astype(np.uint8) - - imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY) - ##imgray = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) - - - ##mask = cv2.inRange(imgray, lower_blue, upper_blue) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - #print(np.unique(mask)) - ##ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - ##plt.imshow(thresh) - ##plt.show() - - contours,hirarchy=cv2.findContours(thresh.copy(), cv2.cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - - areas=[cv2.contourArea(contours[jj]) for jj in range(len(contours))] - - median_area=np.mean(areas) - contours_slope=contours#self.find_polugons_size_filter(contours,median_area=median_area,scaler_up=100,scaler_down=0.5) - - if len(contours_slope)>0: - for jv in range(len(contours_slope)): - new_poly=list(contours_slope[jv]) - if jv==0: - merged_all=new_poly - else: - merged_all=merged_all+new_poly - - - merge=np.array(merged_all) - - - img_in=np.zeros(textline_mask.shape) - img_p_in=cv2.fillPoly(img_in, pts =[merge], color=(255,255,255)) - - ##plt.imshow(img_p_in) - ##plt.show() - - - - rect = cv2.minAreaRect(merge) - - box = cv2.boxPoints(rect) - - box = np.int0(box) - - - indexes=[0,1,2,3] - x_list=box[:,0] - y_list=box[:,1] - - - - index_y_sort=np.argsort(y_list) - - index_upper_left=index_y_sort[np.argmin(x_list[index_y_sort[0:2]])] - index_upper_right=index_y_sort[np.argmax(x_list[index_y_sort[0:2]])] - - index_lower_left=index_y_sort[np.argmin(x_list[index_y_sort[2:]]) +2] - index_lower_right=index_y_sort[np.argmax(x_list[index_y_sort[2:]])+2] - - - alpha1=float(box[index_upper_right][1]- - box[index_upper_left][1])/(float(box[index_upper_right][0]-box[index_upper_left][0])) - alpha2=float(box[index_lower_right][1]- - box[index_lower_left][1])/(float(box[index_lower_right][0]-box[index_lower_left][0])) - - slope_true=(alpha1+alpha2)/2.0 - - #slope=0#slope_true/np.pi*180 - - - #if abs(slope)>=1: - #slope=0 - - #dst=self.rotate_image(textline_mask,slope_true) - #dst=dst[:,:,0] - #dst[dst!=0]=1 - image_regions_deskewd=np.zeros(textline_mask_org[:,:].shape) - for ind in np.unique(textline_mask_org[:,:]): - interest_reg=(textline_mask_org[:,:]==ind)*1 - interest_reg=interest_reg.astype(np.uint8) - deskewed_new=self.rotate_image(interest_reg,slope_true) - deskewed_new=deskewed_new[:,:] - deskewed_new[deskewed_new!=0]=ind - - image_regions_deskewd=image_regions_deskewd+deskewed_new - return image_regions_deskewd,slope_true - - def return_regions_without_seperators(self,regions_pre): - kernel = np.ones((5,5),np.uint8) - regions_without_seperators=( (regions_pre[:,:]!=6) & (regions_pre[:,:]!=0) )*1 - #regions_without_seperators=( (image_regions_eraly_p[:,:,:]!=6) & (image_regions_eraly_p[:,:,:]!=0) & (image_regions_eraly_p[:,:,:]!=5) & (image_regions_eraly_p[:,:,:]!=8) & (image_regions_eraly_p[:,:,:]!=7))*1 - - regions_without_seperators=regions_without_seperators.astype(np.uint8) - - regions_without_seperators = cv2.erode(regions_without_seperators,kernel,iterations = 6) - - return regions_without_seperators - - def return_regions_without_seperators_new(self,regions_pre,regions_only_text): - kernel = np.ones((5,5),np.uint8) - - regions_without_seperators=( (regions_pre[:,:]!=6) & (regions_pre[:,:]!=0) & (regions_pre[:,:]!=1) & (regions_pre[:,:]!=2))*1 - - #plt.imshow(regions_without_seperators) - #plt.show() - - regions_without_seperators_n=( (regions_without_seperators[:,:]==1) | (regions_only_text[:,:]==1) )*1 - - #regions_without_seperators=( (image_regions_eraly_p[:,:,:]!=6) & (image_regions_eraly_p[:,:,:]!=0) & (image_regions_eraly_p[:,:,:]!=5) & (image_regions_eraly_p[:,:,:]!=8) & (image_regions_eraly_p[:,:,:]!=7))*1 - - regions_without_seperators_n=regions_without_seperators_n.astype(np.uint8) - - regions_without_seperators_n = cv2.erode(regions_without_seperators_n,kernel,iterations = 6) - - return regions_without_seperators_n - def image_change_background_pixels_to_zero(self,image_page): - image_back_zero=np.zeros((image_page.shape[0],image_page.shape[1])) - image_back_zero[:,:]=image_page[:,:,0] - image_back_zero[:,:][image_back_zero[:,:]==0]=-255 - image_back_zero[:,:][image_back_zero[:,:]==255]=0 - image_back_zero[:,:][image_back_zero[:,:]==-255]=255 - return image_back_zero - - def find_num_col_only_image(self,regions_without_seperators,multiplier=3.8): - regions_without_seperators_0=regions_without_seperators[:,:].sum(axis=0) - - ##plt.plot(regions_without_seperators_0) - ##plt.show() - - sigma_=15 - - - meda_n_updown=regions_without_seperators_0[len(regions_without_seperators_0)::-1] - - first_nonzero=(next((i for i, x in enumerate(regions_without_seperators_0) if x), 0)) - last_nonzero=(next((i for i, x in enumerate(meda_n_updown) if x), 0)) - - last_nonzero=len(regions_without_seperators_0)-last_nonzero - - - y=regions_without_seperators_0#[first_nonzero:last_nonzero] - - y_help=np.zeros(len(y)+20) - - y_help[10:len(y)+10]=y - - x=np.array( range(len(y)) ) - - - - - zneg_rev=-y_help+np.max(y_help) - - zneg=np.zeros(len(zneg_rev)+20) - - zneg[10:len(zneg_rev)+10]=zneg_rev - - z=gaussian_filter1d(y, sigma_) - zneg= gaussian_filter1d(zneg, sigma_) - - - peaks_neg, _ = find_peaks(zneg, height=0) - peaks, _ = find_peaks(z, height=0) - - - peaks_neg=peaks_neg-10-10 - - peaks_neg_org=np.copy(peaks_neg) - - - peaks_neg=peaks_neg[(peaks_neg>first_nonzero) & (peaks_neg.09*regions_without_seperators.shape[1]) & (peaks<0.91*regions_without_seperators.shape[1])] - - peaks_neg=peaks_neg[ (peaks_neg>500) & (peaks_neg< (regions_without_seperators.shape[1]-500) ) ] - #print(peaks) - interest_pos=z[peaks] - - interest_pos=interest_pos[interest_pos>10] - - interest_neg=z[peaks_neg] - min_peaks_pos=np.mean(interest_pos)#np.min(interest_pos) - min_peaks_neg=0#np.min(interest_neg) - - #$print(min_peaks_pos) - dis_talaei=(min_peaks_pos-min_peaks_neg)/multiplier - #print(interest_pos) - grenze=min_peaks_pos-dis_talaei#np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 - - interest_neg_fin=interest_neg[(interest_negp_g_u and peaks_neg_fin[1]>p_g_u) or (peaks_neg_fin[0]p_m and peaks_neg_fin[1]>p_m ): - num_col=1 - else: - pass - - if num_col==2: - if (peaks_neg_fin[0]>p_g_u) or (peaks_neg_fin[0]cut_off: - #print(forest[np.argmin(z[forest]) ] ) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - forest=[] - forest.append(peaks_neg_fin[i+1]) - if i==(len(peaks_neg_fin)-1): - #print(print(forest[np.argmin(z[forest]) ] )) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - - - - - num_col=(len(peaks_neg_true))+1 - p_l=0 - p_u=len(y)-1 - p_m=int(len(y)/2.) - p_quarter=int(len(y)/4.) - p_g_l=int(len(y)/3.) - p_g_u=len(y)-int(len(y)/3.) - - p_u_quarter=len(y)-p_quarter - - if num_col==3: - if (peaks_neg_true[0]>p_g_u and peaks_neg_true[1]>p_g_u) or (peaks_neg_true[0]p_m and peaks_neg_true[1]>p_m ): - num_col=1 - peaks_neg_true=[] - elif (peaks_neg_true[0]p_g_l) and (peaks_neg_true[1]>p_u_quarter): - peaks_neg_true=[ peaks_neg_true[0] ] - elif (peaks_neg_true[1]p_g_l) and (peaks_neg_true[0]p_g_u) or (peaks_neg_true[0](len(y)-p_g_l)] )==2: - num_col=1 - peaks_neg_true=[] - else: - pass - - - #no deeper hill around found hills - - peaks_fin_true=[] - for i in range(len(peaks_neg_true)): - hill_main=peaks_neg_true[i] - #deep_depth=z[peaks_neg] - hills_around=peaks_neg_org[( (peaks_neg_org>hill_main) & (peaks_neg_org<=hill_main+400) ) | ( (peaks_neg_org=hill_main-400) )] - deep_depth_around=z[hills_around] - - #print(hill_main,z[hill_main],hills_around,deep_depth_around,'manoooo') - try: - if np.min(deep_depth_around)0: - arg_help=np.array(range(len(diff_peaks))) - arg_help_ann=arg_help[diff_peaks<360] - - peaks_neg_fin_new=[] - - for ii in range(len(peaks_neg_fin)): - if ii in arg_help_ann: - arg_min=np.argmin([interest_neg_fin[ii],interest_neg_fin[ii+1] ] ) - if arg_min==0: - peaks_neg_fin_new.append( peaks_neg_fin[ii]) - else: - peaks_neg_fin_new.append( peaks_neg_fin[ii+1]) - - - elif (ii-1) in arg_help_ann: - pass - else: - peaks_neg_fin_new.append(peaks_neg_fin[ii] ) - else: - peaks_neg_fin_new=peaks_neg_fin - - # sometime pages with one columns gives also some negative peaks. delete those peaks - param=z[peaks_neg_true]/float(min_peaks_pos)*100 - - if len(param[param<=41])==0: - peaks_neg_true=[] - - - - - - - return len(peaks_fin_true), peaks_fin_true - - def return_hor_spliter_by_index_for_without_verticals(self,peaks_neg_fin_t,x_min_hor_some,x_max_hor_some): - #print(peaks_neg_fin_t,x_min_hor_some,x_max_hor_some) - arg_min_hor_sort=np.argsort(x_min_hor_some) - x_min_hor_some_sort=np.sort(x_min_hor_some) - x_max_hor_some_sort=x_max_hor_some[arg_min_hor_sort] - - arg_minmax=np.array(range(len(peaks_neg_fin_t))) - indexer_lines=[] - indexes_to_delete=[] - indexer_lines_deletions_len=[] - indexr_uniq_ind=[] - for i in range(len(x_min_hor_some_sort)): - min_h=peaks_neg_fin_t-x_min_hor_some_sort[i] - - - max_h=peaks_neg_fin_t-x_max_hor_some_sort[i] - - - min_h[0]=min_h[0]#+20 - max_h[len(max_h)-1]=max_h[len(max_h)-1]-20 - - - min_h_neg=arg_minmax[(min_h<0)] - min_h_neg_n=min_h[min_h<0] - - try: - min_h_neg=[ min_h_neg[np.argmax(min_h_neg_n)] ] - except: - min_h_neg=[] - - - max_h_neg=arg_minmax[(max_h>0)] - max_h_neg_n=max_h[max_h>0] - - if len(max_h_neg_n)>0: - max_h_neg=[ max_h_neg[np.argmin(max_h_neg_n)] ] - else: - max_h_neg=[] - - - - if len(min_h_neg)>0 and len(max_h_neg)>0: - deletions=list(range(min_h_neg[0]+1,max_h_neg[0])) - unique_delets_int=[] - #print(deletions,len(deletions),'delii') - if len(deletions)>0: - - for j in range(len(deletions)): - indexes_to_delete.append(deletions[j]) - #print(deletions,indexes_to_delete,'badiii') - unique_delets=np.unique(indexes_to_delete) - #print(min_h_neg[0],unique_delets) - unique_delets_int=unique_delets[unique_delets0 and min_h_neg[0]>=2: - index_line_true=index_line_true - else: - index_line_true=min_h_neg[0] - - indexer_lines.append(index_line_true) - - if len(unique_delets_int)>0: - for dd in range(len(unique_delets_int)): - indexes_to_delete.append(unique_delets_int[dd]) - else: - indexer_lines.append(-999) - indexer_lines_deletions_len.append(-999) - indexr_uniq_ind.append(-999) - - peaks_true=[] - for m in range(len(peaks_neg_fin_t)): - if m in indexes_to_delete: - pass - else: - peaks_true.append(peaks_neg_fin_t[m]) - return indexer_lines,peaks_true,arg_min_hor_sort,indexer_lines_deletions_len,indexr_uniq_ind - - - def find_num_col_by_vertical_lines(self,regions_without_seperators,multiplier=3.8): - regions_without_seperators_0=regions_without_seperators[:,:,0].sum(axis=0) - - ##plt.plot(regions_without_seperators_0) - ##plt.show() - - sigma_=35#70#35 - - - - z=gaussian_filter1d(regions_without_seperators_0, sigma_) - - peaks, _ = find_peaks(z, height=0) - - #print(peaks,'peaksnew') - return peaks - - - def find_num_col(self,regions_without_seperators,multiplier=3.8): - regions_without_seperators_0=regions_without_seperators[:,:].sum(axis=0) - - ##plt.plot(regions_without_seperators_0) - ##plt.show() - - sigma_=35#70#35 - - - meda_n_updown=regions_without_seperators_0[len(regions_without_seperators_0)::-1] - - first_nonzero=(next((i for i, x in enumerate(regions_without_seperators_0) if x), 0)) - last_nonzero=(next((i for i, x in enumerate(meda_n_updown) if x), 0)) - - #print(last_nonzero) - #print(isNaN(last_nonzero)) - #last_nonzero=0#halalikh - last_nonzero=len(regions_without_seperators_0)-last_nonzero - - - y=regions_without_seperators_0#[first_nonzero:last_nonzero] - - y_help=np.zeros(len(y)+20) - - y_help[10:len(y)+10]=y - - x=np.array( range(len(y)) ) - - - - - zneg_rev=-y_help+np.max(y_help) - - zneg=np.zeros(len(zneg_rev)+20) - - zneg[10:len(zneg_rev)+10]=zneg_rev - - z=gaussian_filter1d(y, sigma_) - zneg= gaussian_filter1d(zneg, sigma_) - - - peaks_neg, _ = find_peaks(zneg, height=0) - peaks, _ = find_peaks(z, height=0) - - peaks_neg=peaks_neg-10-10 - - - last_nonzero=last_nonzero-100 - first_nonzero=first_nonzero+200 - - peaks_neg=peaks_neg[(peaks_neg>first_nonzero) & (peaks_neg.06*regions_without_seperators.shape[1]) & (peaks<0.94*regions_without_seperators.shape[1])] - peaks_neg=peaks_neg[ (peaks_neg>370) & (peaks_neg< (regions_without_seperators.shape[1]-370) ) ] - - #print(peaks) - interest_pos=z[peaks] - - interest_pos=interest_pos[interest_pos>10] - - - - #plt.plot(z) - #plt.show() - interest_neg=z[peaks_neg] - - - - - min_peaks_pos=np.min(interest_pos) - max_peaks_pos=np.max(interest_pos) - - if max_peaks_pos/min_peaks_pos>=35: - min_peaks_pos=np.mean(interest_pos) - - min_peaks_neg=0#np.min(interest_neg) - - #print(np.min(interest_pos),np.max(interest_pos),np.max(interest_pos)/np.min(interest_pos),'minmax') - #$print(min_peaks_pos) - dis_talaei=(min_peaks_pos-min_peaks_neg)/multiplier - #print(interest_pos) - grenze=min_peaks_pos-dis_talaei#np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 - - #print(interest_neg,'interest_neg') - #print(grenze,'grenze') - #print(min_peaks_pos,'min_peaks_pos') - #print(dis_talaei,'dis_talaei') - #print(peaks_neg,'peaks_neg') - - interest_neg_fin=interest_neg[(interest_negp_g_u and peaks_neg_fin[1]>p_g_u) or (peaks_neg_fin[0]p_m and peaks_neg_fin[1]>p_m ): - num_col=1 - peaks_neg_fin=[] - else: - pass - - if num_col==2: - if (peaks_neg_fin[0]>p_g_u) or (peaks_neg_fin[0]cut_off: - #print(forest[np.argmin(z[forest]) ] ) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - forest=[] - forest.append(peaks_neg_fin[i+1]) - if i==(len(peaks_neg_fin)-1): - #print(print(forest[np.argmin(z[forest]) ] )) - if not self.isNaN(forest[np.argmin(z[forest]) ]): - peaks_neg_true.append(forest[np.argmin(z[forest]) ]) - - - num_col=(len(peaks_neg_true))+1 - p_l=0 - p_u=len(y)-1 - p_m=int(len(y)/2.) - p_quarter=int(len(y)/5.) - p_g_l=int(len(y)/4.) - p_g_u=len(y)-int(len(y)/4.) - - p_u_quarter=len(y)-p_quarter - - ##print(num_col,'early') - if num_col==3: - if (peaks_neg_true[0]>p_g_u and peaks_neg_true[1]>p_g_u) or (peaks_neg_true[0]p_m and peaks_neg_true[1]>p_m ): - num_col=1 - peaks_neg_true=[] - elif (peaks_neg_true[0]p_g_l) and (peaks_neg_true[1]>p_u_quarter): - peaks_neg_true=[ peaks_neg_true[0] ] - elif (peaks_neg_true[1]p_g_l) and (peaks_neg_true[0]p_g_u) or (peaks_neg_true[0]0: - arg_help=np.array(range(len(diff_peaks))) - arg_help_ann=arg_help[diff_peaks<360] - - peaks_neg_fin_new=[] - - for ii in range(len(peaks_neg_fin)): - if ii in arg_help_ann: - arg_min=np.argmin([interest_neg_fin[ii],interest_neg_fin[ii+1] ] ) - if arg_min==0: - peaks_neg_fin_new.append( peaks_neg_fin[ii]) - else: - peaks_neg_fin_new.append( peaks_neg_fin[ii+1]) - - - elif (ii-1) in arg_help_ann: - pass - else: - peaks_neg_fin_new.append(peaks_neg_fin[ii] ) - else: - peaks_neg_fin_new=peaks_neg_fin - - - #plt.plot(gaussian_filter1d(y, sigma_)) - #plt.plot(peaks_neg_true,z[peaks_neg_true],'*') - #plt.plot([0,len(y)], [grenze,grenze]) - #plt.show() - - ##print(len(peaks_neg_true)) - return len(peaks_neg_true), peaks_neg_true - - def find_new_features_of_contoures(self,contours_main): - - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - try: - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - argmin_x_main=np.array([np.argmin(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - x_min_from_argmin=np.array([contours_main[j][argmin_x_main[j],0,0] for j in range(len(contours_main))]) - y_corr_x_min_from_argmin=np.array([contours_main[j][argmin_x_main[j],0,1] for j in range(len(contours_main))]) - - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - except: - x_min_main=np.array([np.min(contours_main[j][:,0]) for j in range(len(contours_main))]) - - argmin_x_main=np.array([np.argmin(contours_main[j][:,0]) for j in range(len(contours_main))]) - - - x_min_from_argmin=np.array([contours_main[j][argmin_x_main[j],0] for j in range(len(contours_main))]) - y_corr_x_min_from_argmin=np.array([contours_main[j][argmin_x_main[j],1] for j in range(len(contours_main))]) - - x_max_main=np.array([np.max(contours_main[j][:,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,1]) for j in range(len(contours_main))]) - - - #dis_x=np.abs(x_max_main-x_min_main) - - return cx_main,cy_main ,x_min_main , x_max_main, y_min_main ,y_max_main,y_corr_x_min_from_argmin - def return_points_with_boundies(self,peaks_neg_fin,first_point, last_point): - peaks_neg_tot=[] - peaks_neg_tot.append(first_point) - for ii in range(len(peaks_neg_fin)): - peaks_neg_tot.append(peaks_neg_fin[ii]) - peaks_neg_tot.append(last_point) - return peaks_neg_tot - def contours_in_same_horizon(self,cy_main_hor): - X1=np.zeros((len(cy_main_hor),len(cy_main_hor))) - X2=np.zeros((len(cy_main_hor),len(cy_main_hor))) - - X1[0::1,:]=cy_main_hor[:] - X2=X1.T - - X_dif=np.abs(X2-X1) - args_help=np.array(range(len(cy_main_hor))) - all_args=[] - for i in range(len(cy_main_hor)): - list_h=list(args_help[X_dif[i,:]<=20] ) - list_h.append(i) - if len(list_h)>1: - all_args.append(list( set(list_h) )) - return np.unique(all_args) - - def return_boxes_of_images_by_order_of_reading_without_seperators(self,spliter_y_new,image_p_rev,regions_without_seperators,matrix_of_lines_ch,seperators_closeup_n): - - boxes=[] - - - # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ - # holes in the text and also finding spliter which covers more than one columns. - for i in range(len(spliter_y_new)-1): - #print(spliter_y_new[i],spliter_y_new[i+1]) - matrix_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,6]> spliter_y_new[i] ) & (matrix_of_lines_ch[:,7]< spliter_y_new[i+1] ) ] - #print(len( matrix_new[:,9][matrix_new[:,9]==1] )) - - #print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') - - # check to see is there any vertical seperator to find holes. - if np.abs(spliter_y_new[i+1]-spliter_y_new[i])>1./3.*regions_without_seperators.shape[0]:#len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): - - #org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) - #org_img_dichte=org_img_dichte-np.min(org_img_dichte) - ##plt.figure(figsize=(20,20)) - ##plt.plot(org_img_dichte) - ##plt.show() - ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) - - num_col, peaks_neg_fin=self.find_num_col_only_image(image_p_rev[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=2.4) - - #num_col, peaks_neg_fin=find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) - x_min_hor_some=matrix_new[:,2][ (matrix_new[:,9]==0) ] - x_max_hor_some=matrix_new[:,3][ (matrix_new[:,9]==0) ] - cy_hor_some=matrix_new[:,5][ (matrix_new[:,9]==0) ] - arg_org_hor_some=matrix_new[:,0][ (matrix_new[:,9]==0) ] - - - - peaks_neg_tot=self.return_points_with_boundies(peaks_neg_fin,0, seperators_closeup_n[:,:,0].shape[1]) - - start_index_of_hor,newest_peaks,arg_min_hor_sort,lines_length_dels,lines_indexes_deleted=self.return_hor_spliter_by_index_for_without_verticals(peaks_neg_tot,x_min_hor_some,x_max_hor_some) - - arg_org_hor_some_sort=arg_org_hor_some[arg_min_hor_sort] - - - start_index_of_hor_with_subset=[start_index_of_hor[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ]#start_index_of_hor[lines_length_dels>0] - arg_min_hor_sort_with_subset=[arg_min_hor_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - lines_indexes_deleted_with_subset=[lines_indexes_deleted[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - lines_length_dels_with_subset=[lines_length_dels[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - - arg_org_hor_some_sort_subset=[arg_org_hor_some_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - - #arg_min_hor_sort_with_subset=arg_min_hor_sort[lines_length_dels>0] - #lines_indexes_deleted_with_subset=lines_indexes_deleted[lines_length_dels>0] - #lines_length_dels_with_subset=lines_length_dels[lines_length_dels>0] - - - - - #print(len(arg_min_hor_sort),len(arg_org_hor_some_sort),'vizzzzzz') - - - - vahid_subset=np.zeros((len(start_index_of_hor_with_subset),len(start_index_of_hor_with_subset)))-1 - for kkk1 in range(len(start_index_of_hor_with_subset)): - - #print(lines_indexes_deleted,'hiii') - index_del_sub=np.unique(lines_indexes_deleted_with_subset[kkk1]) - - for kkk2 in range(len(start_index_of_hor_with_subset)): - - if set(lines_indexes_deleted_with_subset[kkk2][0]) < set(lines_indexes_deleted_with_subset[kkk1][0]): - vahid_subset[kkk1,kkk2]=kkk1 - else: - pass - #print(set(lines_indexes_deleted[kkk2][0]), set(lines_indexes_deleted[kkk1][0])) - - - - # check the len of matrix if it has no length means that there is no spliter at all - - if len(vahid_subset>0): - #print('hihoo') - - - # find parenets args - line_int=np.zeros(vahid_subset.shape[0]) - - childs_id=[] - arg_child=[] - for li in range(vahid_subset.shape[0]): - if np.all(vahid_subset[:,li]==-1): - line_int[li]=-1 - else: - line_int[li]=1 - - - #childs_args_in=[ idd for idd in range(vahid_subset.shape[0]) if vahid_subset[idd,li]!=-1] - #helpi=[] - #for nad in range(len(childs_args_in)): - # helpi.append(arg_min_hor_sort_with_subset[childs_args_in[nad]]) - - - arg_child.append(arg_min_hor_sort_with_subset[li] ) - - - - - arg_parent=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - start_index_of_hor_parent=[start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - #arg_parent=[lines_indexes_deleted_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - #arg_parent=[lines_length_dels_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - - - #arg_child=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] - start_index_of_hor_child=[start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] - - - cy_hor_some_sort=cy_hor_some[arg_parent] - - - newest_y_spliter_tot=[] - - for tj in range(len(newest_peaks)-1): - newest_y_spliter=[] - newest_y_spliter.append(spliter_y_new[i]) - if tj in np.unique(start_index_of_hor_parent): - cy_help=np.array(cy_hor_some_sort)[np.array(start_index_of_hor_parent)==tj] - cy_help_sort=np.sort(cy_help) - - #print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') - for mj in range(len(cy_help_sort)): - newest_y_spliter.append(cy_help_sort[mj]) - newest_y_spliter.append(spliter_y_new[i+1]) - - newest_y_spliter_tot.append(newest_y_spliter) - - - - - else: - line_int=[] - newest_y_spliter_tot=[] - - for tj in range(len(newest_peaks)-1): - newest_y_spliter=[] - newest_y_spliter.append(spliter_y_new[i]) - - newest_y_spliter.append(spliter_y_new[i+1]) - - newest_y_spliter_tot.append(newest_y_spliter) - - - - # if line_int is all -1 means that big spliters have no child and we can easily go through - if np.all(np.array(line_int)==-1): - for j in range(len(newest_peaks)-1): - newest_y_spliter=newest_y_spliter_tot[j] - - - for n in range(len(newest_y_spliter)-1): - #print(j,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'maaaa') - ##plt.imshow(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]]) - ##plt.show() - - #print(matrix_new[:,0][ (matrix_new[:,9]==1 )]) - for jvt in matrix_new[:,0][ (matrix_new[:,9]==1 ) & (matrix_new[:,6]> newest_y_spliter[n] ) & (matrix_new[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_new[:,1]) < newest_peaks[j+1] ) & (( matrix_new[:,1])> newest_peaks[j] ) ] : - pass - - ###plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) - #print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - #num_col_sub, peaks_neg_fin_sub=find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) - num_col_sub, peaks_neg_fin_sub=self.find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.4) - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - else: - for j in range(len(newest_peaks)-1): - newest_y_spliter=newest_y_spliter_tot[j] - - if j in start_index_of_hor_parent: - - x_min_ch=x_min_hor_some[arg_child] - x_max_ch=x_max_hor_some[arg_child] - cy_hor_some_sort_child=cy_hor_some[arg_child] - cy_hor_some_sort_child=np.sort(cy_hor_some_sort_child) - - for n in range(len(newest_y_spliter)-1): - - cy_child_in=cy_hor_some_sort_child[( cy_hor_some_sort_child>newest_y_spliter[n] ) & ( cy_hor_some_sort_child0: - ###num_col_ch, peaks_neg_ch=find_num_col( regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) - - num_col_ch, peaks_neg_ch=self.find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) - - peaks_neg_ch=peaks_neg_ch[:]+newest_peaks[j] - - peaks_neg_ch_tot=self.return_points_with_boundies(peaks_neg_ch,newest_peaks[j], newest_peaks[j+1]) - - - ss_in_ch,nst_p_ch,arg_n_ch,lines_l_del_ch,lines_in_del_ch=self.return_hor_spliter_by_index_for_without_verticals(peaks_neg_ch_tot,x_min_ch,x_max_ch) - - newest_y_spliter_ch_tot=[] - - for tjj in range(len(nst_p_ch)-1): - newest_y_spliter_new=[] - newest_y_spliter_new.append(newest_y_spliter[n]) - if tjj in np.unique(ss_in_ch): - - - #print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') - for mjj in range(len(cy_child_in)): - newest_y_spliter_new.append(cy_child_in[mjj]) - newest_y_spliter_new.append(newest_y_spliter[n+1]) - - newest_y_spliter_ch_tot.append(newest_y_spliter_new) - - - - - for jn in range(len(nst_p_ch)-1): - newest_y_spliter_h=newest_y_spliter_ch_tot[jn] - - for nd in range(len(newest_y_spliter_h)-1): - - matrix_new_new2=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter_h[nd] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter_h[nd+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < nst_p_ch[jn+1] ) & (( matrix_of_lines_ch[:,1]-500)>nst_p_ch[jn] ) ] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new2[:,9][matrix_new_new2[:,9]==1] )>0 and np.max(matrix_new_new2[:,8][matrix_new_new2[:,9]==1])>=0.2*(np.abs(newest_y_spliter_h[nd+1]-newest_y_spliter_h[nd] )): - #num_col_sub_ch, peaks_neg_fin_sub_ch=find_num_col(regions_without_seperators[int(newest_y_spliter_h[nd]):int(newest_y_spliter_h[nd+1]),nst_p_ch[jn]:nst_p_ch[jn+1]],multiplier=2.3) - - num_col_sub_ch, peaks_neg_fin_sub_ch=self.find_num_col_only_image(image_p_rev[int(newest_y_spliter_h[nd]):int(newest_y_spliter_h[nd+1]),nst_p_ch[jn]:nst_p_ch[jn+1]],multiplier=2.3) - #print(peaks_neg_fin_sub_ch,'gada kutullllllll') - else: - peaks_neg_fin_sub_ch=[] - - peaks_sub_ch=[] - peaks_sub_ch.append(nst_p_ch[jn]) - - for kjj in range(len(peaks_neg_fin_sub_ch)): - peaks_sub_ch.append(peaks_neg_fin_sub_ch[kjj]+nst_p_ch[jn]) - - peaks_sub_ch.append(nst_p_ch[jn+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for khh in range(len(peaks_sub_ch)-1): - boxes.append([ peaks_sub_ch[khh], peaks_sub_ch[khh+1] ,newest_y_spliter_h[nd],newest_y_spliter_h[nd+1]]) - - - - else: - - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - ###num_col_sub, peaks_neg_fin_sub=find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) - num_col_sub, peaks_neg_fin_sub=self.find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - - - - else: - for n in range(len(newest_y_spliter)-1): - - for jvt in matrix_new[:,0][ (matrix_new[:,9]==1 ) & (matrix_new[:,6]> newest_y_spliter[n] ) & (matrix_new[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_new[:,1]) < newest_peaks[j+1] ) & (( matrix_new[:,1])> newest_peaks[j] ) ] : - pass - - #plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) - #print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - ###num_col_sub, peaks_neg_fin_sub=find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.0) - num_col_sub, peaks_neg_fin_sub=self.find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - - - - - - - else: - boxes.append([ 0, seperators_closeup_n[:,:,0].shape[1] ,spliter_y_new[i],spliter_y_new[i+1]]) - return boxes - - def return_boxes_of_images_by_order_of_reading_without_seperators_2cols(self,spliter_y_new,image_p_rev,regions_without_seperators,matrix_of_lines_ch,seperators_closeup_n): - - boxes=[] - - - # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ - # holes in the text and also finding spliter which covers more than one columns. - for i in range(len(spliter_y_new)-1): - #print(spliter_y_new[i],spliter_y_new[i+1]) - matrix_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,6]> spliter_y_new[i] ) & (matrix_of_lines_ch[:,7]< spliter_y_new[i+1] ) ] - #print(len( matrix_new[:,9][matrix_new[:,9]==1] )) - - #print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') - - # check to see is there any vertical seperator to find holes. - if np.abs(spliter_y_new[i+1]-spliter_y_new[i])>1./3.*regions_without_seperators.shape[0]:#len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): - - #org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) - #org_img_dichte=org_img_dichte-np.min(org_img_dichte) - ##plt.figure(figsize=(20,20)) - ##plt.plot(org_img_dichte) - ##plt.show() - ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) - - try: - num_col, peaks_neg_fin=self.find_num_col_only_image(image_p_rev[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=2.4) - except: - peaks_neg_fin=[] - num_col=0 - - peaks_neg_tot=self.return_points_with_boundies(peaks_neg_fin,0, seperators_closeup_n[:,:,0].shape[1]) - - for kh in range(len(peaks_neg_tot)-1): - boxes.append([ peaks_neg_tot[kh], peaks_neg_tot[kh+1] ,spliter_y_new[i],spliter_y_new[i+1]]) - else: - boxes.append([ 0, seperators_closeup_n[:,:,0].shape[1] ,spliter_y_new[i],spliter_y_new[i+1]]) - - - - - - - return boxes - def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back(self, regions_pre_p): - seperators_closeup=( (regions_pre_p[:,:]==6))*1 - - seperators_closeup=seperators_closeup.astype(np.uint8) - kernel = np.ones((5,5),np.uint8) - - - seperators_closeup = cv2.dilate(seperators_closeup,kernel,iterations = 1) - seperators_closeup = cv2.erode(seperators_closeup,kernel,iterations = 1) - - seperators_closeup = cv2.erode(seperators_closeup,kernel,iterations = 1) - seperators_closeup = cv2.dilate(seperators_closeup,kernel,iterations = 1) - - if len(seperators_closeup.shape)==2: - seperators_closeup_n=np.zeros((seperators_closeup.shape[0],seperators_closeup.shape[1],3)) - seperators_closeup_n[:,:,0]=seperators_closeup - seperators_closeup_n[:,:,1]=seperators_closeup - seperators_closeup_n[:,:,2]=seperators_closeup - else: - seperators_closeup_n=seperators_closeup[:,:,:] - #seperators_closeup=seperators_closeup.astype(np.uint8) - seperators_closeup_n=seperators_closeup_n.astype(np.uint8) - imgray = cv2.cvtColor(seperators_closeup_n, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_lines,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=self.find_features_of_lines(contours_lines) - - dist_y=np.abs(y_max_main-y_min_main) - - slope_lines_org_hor=slope_lines_org[slope_lines==0] - args=np.array( range(len(slope_lines) )) - len_x=seperators_closeup.shape[1]*0 - len_y=seperators_closeup.shape[0]*.01 - - args_hor=args[slope_lines==0] - dist_x_hor=dist_x[slope_lines==0] - dist_y_hor=dist_y[slope_lines==0] - x_min_main_hor=x_min_main[slope_lines==0] - x_max_main_hor=x_max_main[slope_lines==0] - cy_main_hor=cy_main[slope_lines==0] - y_min_main_hor=y_min_main[slope_lines==0] - y_max_main_hor=y_max_main[slope_lines==0] - - args_hor=args_hor[dist_x_hor>=len_x] - x_max_main_hor=x_max_main_hor[dist_x_hor>=len_x] - x_min_main_hor=x_min_main_hor[dist_x_hor>=len_x] - cy_main_hor=cy_main_hor[dist_x_hor>=len_x] - y_min_main_hor=y_min_main_hor[dist_x_hor>=len_x] - y_max_main_hor=y_max_main_hor[dist_x_hor>=len_x] - slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x] - dist_y_hor=dist_y_hor[dist_x_hor>=len_x] - dist_x_hor=dist_x_hor[dist_x_hor>=len_x] - - - args_ver=args[slope_lines==1] - dist_y_ver=dist_y[slope_lines==1] - dist_x_ver=dist_x[slope_lines==1] - x_min_main_ver=x_min_main[slope_lines==1] - x_max_main_ver=x_max_main[slope_lines==1] - y_min_main_ver=y_min_main[slope_lines==1] - y_max_main_ver=y_max_main[slope_lines==1] - cx_main_ver=cx_main[slope_lines==1] - - args_ver=args_ver[dist_y_ver>=len_y] - x_max_main_ver=x_max_main_ver[dist_y_ver>=len_y] - x_min_main_ver=x_min_main_ver[dist_y_ver>=len_y] - cx_main_ver=cx_main_ver[dist_y_ver>=len_y] - y_min_main_ver=y_min_main_ver[dist_y_ver>=len_y] - y_max_main_ver=y_max_main_ver[dist_y_ver>=len_y] - dist_x_ver=dist_x_ver[dist_y_ver>=len_y] - dist_y_ver=dist_y_ver[dist_y_ver>=len_y] - - img_p_in_ver=np.zeros(seperators_closeup_n[:,:,2].shape) - for jv in range(len(args_ver)): - img_p_in_ver=cv2.fillPoly(img_p_in_ver, pts =[contours_lines[args_ver[jv]]], color=(1,1,1)) - - img_in_hor=np.zeros(seperators_closeup_n[:,:,2].shape) - for jv in range(len(args_hor)): - img_p_in_hor=cv2.fillPoly(img_in_hor, pts =[contours_lines[args_hor[jv]]], color=(1,1,1)) - - all_args_uniq=self.contours_in_same_horizon(cy_main_hor) - #print(all_args_uniq,'all_args_uniq') - if len(all_args_uniq)>0: - if type(all_args_uniq[0]) is list: - contours_new=[] - for dd in range(len(all_args_uniq)): - merged_all=None - some_args=args_hor[all_args_uniq[dd]] - some_cy=cy_main_hor[all_args_uniq[dd]] - some_x_min=x_min_main_hor[all_args_uniq[dd]] - some_x_max=x_max_main_hor[all_args_uniq[dd]] - - img_in=np.zeros(seperators_closeup_n[:,:,2].shape) - for jv in range(len(some_args)): - - img_p_in=cv2.fillPoly(img_p_in_hor, pts =[contours_lines[some_args[jv]]], color=(1,1,1)) - img_p_in[int(np.mean(some_cy))-5:int(np.mean(some_cy))+5, int(np.min(some_x_min)):int(np.max(some_x_max)) ]=1 - - else: - img_p_in=seperators_closeup - else: - img_p_in=seperators_closeup - - sep_ver_hor=img_p_in+img_p_in_ver - sep_ver_hor_cross=(sep_ver_hor==2)*1 - - sep_ver_hor_cross=np.repeat(sep_ver_hor_cross[:, :, np.newaxis], 3, axis=2) - sep_ver_hor_cross=sep_ver_hor_cross.astype(np.uint8) - imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_cross,_=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - cx_cross,cy_cross ,_ , _, _ ,_,_=self.find_new_features_of_contoures(contours_cross) - - for ii in range(len(cx_cross)): - sep_ver_hor[int(cy_cross[ii])-15:int(cy_cross[ii])+15,int(cx_cross[ii])+5:int(cx_cross[ii])+40]=0 - sep_ver_hor[int(cy_cross[ii])-15:int(cy_cross[ii])+15,int(cx_cross[ii])-40:int(cx_cross[ii])-4]=0 - - img_p_in[:,:]=sep_ver_hor[:,:] - - if len(img_p_in.shape)==2: - seperators_closeup_n=np.zeros((img_p_in.shape[0],img_p_in.shape[1],3)) - seperators_closeup_n[:,:,0]=img_p_in - seperators_closeup_n[:,:,1]=img_p_in - seperators_closeup_n[:,:,2]=img_p_in - else: - seperators_closeup_n=img_p_in[:,:,:] - #seperators_closeup=seperators_closeup.astype(np.uint8) - seperators_closeup_n=seperators_closeup_n.astype(np.uint8) - imgray = cv2.cvtColor(seperators_closeup_n, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_lines,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=self.find_features_of_lines(contours_lines) - - dist_y=np.abs(y_max_main-y_min_main) - - slope_lines_org_hor=slope_lines_org[slope_lines==0] - args=np.array( range(len(slope_lines) )) - len_x=seperators_closeup.shape[1]*.04 - len_y=seperators_closeup.shape[0]*.08 - - args_hor=args[slope_lines==0] - dist_x_hor=dist_x[slope_lines==0] - dist_y_hor=dist_y[slope_lines==0] - x_min_main_hor=x_min_main[slope_lines==0] - x_max_main_hor=x_max_main[slope_lines==0] - cy_main_hor=cy_main[slope_lines==0] - y_min_main_hor=y_min_main[slope_lines==0] - y_max_main_hor=y_max_main[slope_lines==0] - - args_hor=args_hor[dist_x_hor>=len_x] - x_max_main_hor=x_max_main_hor[dist_x_hor>=len_x] - x_min_main_hor=x_min_main_hor[dist_x_hor>=len_x] - cy_main_hor=cy_main_hor[dist_x_hor>=len_x] - y_min_main_hor=y_min_main_hor[dist_x_hor>=len_x] - y_max_main_hor=y_max_main_hor[dist_x_hor>=len_x] - slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x] - dist_y_hor=dist_y_hor[dist_x_hor>=len_x] - dist_x_hor=dist_x_hor[dist_x_hor>=len_x] - - - - args_ver=args[slope_lines==1] - dist_y_ver=dist_y[slope_lines==1] - dist_x_ver=dist_x[slope_lines==1] - x_min_main_ver=x_min_main[slope_lines==1] - x_max_main_ver=x_max_main[slope_lines==1] - y_min_main_ver=y_min_main[slope_lines==1] - y_max_main_ver=y_max_main[slope_lines==1] - cx_main_ver=cx_main[slope_lines==1] - - args_ver=args_ver[dist_y_ver>=len_y] - x_max_main_ver=x_max_main_ver[dist_y_ver>=len_y] - x_min_main_ver=x_min_main_ver[dist_y_ver>=len_y] - cx_main_ver=cx_main_ver[dist_y_ver>=len_y] - y_min_main_ver=y_min_main_ver[dist_y_ver>=len_y] - y_max_main_ver=y_max_main_ver[dist_y_ver>=len_y] - dist_x_ver=dist_x_ver[dist_y_ver>=len_y] - dist_y_ver=dist_y_ver[dist_y_ver>=len_y] - - matrix_of_lines_ch=np.zeros((len(cy_main_hor)+len(cx_main_ver),10)) - - matrix_of_lines_ch[:len(cy_main_hor),0]=args_hor - matrix_of_lines_ch[len(cy_main_hor):,0]=args_ver - - - matrix_of_lines_ch[len(cy_main_hor):,1]=cx_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),2]=x_min_main_hor - matrix_of_lines_ch[len(cy_main_hor):,2]=x_min_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),3]=x_max_main_hor - matrix_of_lines_ch[len(cy_main_hor):,3]=x_max_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),4]=dist_x_hor - matrix_of_lines_ch[len(cy_main_hor):,4]=dist_x_ver - - matrix_of_lines_ch[:len(cy_main_hor),5]=cy_main_hor - - - matrix_of_lines_ch[:len(cy_main_hor),6]=y_min_main_hor - matrix_of_lines_ch[len(cy_main_hor):,6]=y_min_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),7]=y_max_main_hor - matrix_of_lines_ch[len(cy_main_hor):,7]=y_max_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),8]=dist_y_hor - matrix_of_lines_ch[len(cy_main_hor):,8]=dist_y_ver - - - matrix_of_lines_ch[len(cy_main_hor):,9]=1 - - return matrix_of_lines_ch,seperators_closeup_n - - - def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new(self, img_p_in_ver,img_in_hor): - - - #plt.imshow(img_in_hor) - #plt.show() - - #img_p_in_ver = cv2.erode(img_p_in_ver, self.kernel, iterations=2) - img_p_in_ver=img_p_in_ver.astype(np.uint8) - img_p_in_ver=np.repeat(img_p_in_ver[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(img_p_in_ver, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_lines_ver,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - slope_lines_ver,dist_x_ver, x_min_main_ver ,x_max_main_ver ,cy_main_ver,slope_lines_org_ver,y_min_main_ver, y_max_main_ver, cx_main_ver=self.find_features_of_lines(contours_lines_ver) - - for i in range(len(x_min_main_ver)): - img_p_in_ver[int(y_min_main_ver[i]):int(y_min_main_ver[i])+30,int(cx_main_ver[i])-25:int(cx_main_ver[i])+25,0]=0 - img_p_in_ver[int(y_max_main_ver[i])-30:int(y_max_main_ver[i]),int(cx_main_ver[i])-25:int(cx_main_ver[i])+25,0]=0 - - - #plt.imshow(img_p_in_ver[:,:,0]) - #plt.show() - img_in_hor=img_in_hor.astype(np.uint8) - img_in_hor=np.repeat(img_in_hor[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(img_in_hor, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_lines_hor,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - slope_lines_hor,dist_x_hor, x_min_main_hor ,x_max_main_hor ,cy_main_hor,slope_lines_org_hor,y_min_main_hor, y_max_main_hor, cx_main_hor=self.find_features_of_lines(contours_lines_hor) - - args_hor=np.array( range(len(slope_lines_hor) )) - all_args_uniq=self.contours_in_same_horizon(cy_main_hor) - #print(all_args_uniq,'all_args_uniq') - if len(all_args_uniq)>0: - if type(all_args_uniq[0]) is list: - special_seperators=[] - contours_new=[] - for dd in range(len(all_args_uniq)): - merged_all=None - some_args=args_hor[all_args_uniq[dd]] - some_cy=cy_main_hor[all_args_uniq[dd]] - some_x_min=x_min_main_hor[all_args_uniq[dd]] - some_x_max=x_max_main_hor[all_args_uniq[dd]] - - #img_in=np.zeros(seperators_closeup_n[:,:,2].shape) - for jv in range(len(some_args)): - - img_p_in=cv2.fillPoly(img_in_hor, pts =[contours_lines_hor[some_args[jv]]], color=(1,1,1)) - img_p_in[int(np.mean(some_cy))-5:int(np.mean(some_cy))+5, int(np.min(some_x_min)):int(np.max(some_x_max)) ]=1 - - sum_dis=dist_x_hor[some_args].sum() - diff_max_min_uniques=np.max(x_max_main_hor[some_args])-np.min(x_min_main_hor[some_args]) - - - #print( sum_dis/float(diff_max_min_uniques) ,diff_max_min_uniques/float(img_p_in_ver.shape[1]),dist_x_hor[some_args].sum(),diff_max_min_uniques,np.mean( dist_x_hor[some_args]),np.std( dist_x_hor[some_args]) ) - - if diff_max_min_uniques>sum_dis and ( (sum_dis/float(diff_max_min_uniques) ) >0.85 ) and ( (diff_max_min_uniques/float(img_p_in_ver.shape[1]))>0.85 ) and np.std( dist_x_hor[some_args] )<(0.55*np.mean( dist_x_hor[some_args] )): - #print(dist_x_hor[some_args],dist_x_hor[some_args].sum(),np.min(x_min_main_hor[some_args]) ,np.max(x_max_main_hor[some_args]),'jalibdi') - #print(np.mean( dist_x_hor[some_args] ),np.std( dist_x_hor[some_args] ),np.var( dist_x_hor[some_args] ),'jalibdiha') - special_seperators.append(np.mean(cy_main_hor[some_args])) - - else: - img_p_in=img_in_hor - special_seperators=[] - else: - img_p_in=img_in_hor - special_seperators=[] - - - img_p_in_ver[:,:,0][img_p_in_ver[:,:,0]==255]=1 - #print(img_p_in_ver.shape,np.unique(img_p_in_ver[:,:,0])) - - #plt.imshow(img_p_in[:,:,0]) - #plt.show() - - #plt.imshow(img_p_in_ver[:,:,0]) - #plt.show() - sep_ver_hor=img_p_in+img_p_in_ver - #print(sep_ver_hor.shape,np.unique(sep_ver_hor[:,:,0]),'sep_ver_horsep_ver_horsep_ver_hor') - #plt.imshow(sep_ver_hor[:,:,0]) - #plt.show() - - sep_ver_hor_cross=(sep_ver_hor[:,:,0]==2)*1 - - sep_ver_hor_cross=np.repeat(sep_ver_hor_cross[:, :, np.newaxis], 3, axis=2) - sep_ver_hor_cross=sep_ver_hor_cross.astype(np.uint8) - imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_cross,_=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - cx_cross,cy_cross ,_ , _, _ ,_,_=self.find_new_features_of_contoures(contours_cross) - - for ii in range(len(cx_cross)): - img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])+5:int(cx_cross[ii])+40,0]=0 - img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])-40:int(cx_cross[ii])-4,0]=0 - - #plt.imshow(img_p_in[:,:,0]) - #plt.show() - - return img_p_in[:,:,0],special_seperators - - def return_boxes_of_images_by_order_of_reading(self,spliter_y_new,regions_without_seperators,matrix_of_lines_ch,seperators_closeup_n): - boxes=[] - - - # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ - # holes in the text and also finding spliter which covers more than one columns. - for i in range(len(spliter_y_new)-1): - #print(spliter_y_new[i],spliter_y_new[i+1]) - matrix_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,6]> spliter_y_new[i] ) & (matrix_of_lines_ch[:,7]< spliter_y_new[i+1] ) ] - #print(len( matrix_new[:,9][matrix_new[:,9]==1] )) - - #print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') - - # check to see is there any vertical seperator to find holes. - if len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): - - #org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) - #org_img_dichte=org_img_dichte-np.min(org_img_dichte) - ##plt.figure(figsize=(20,20)) - ##plt.plot(org_img_dichte) - ##plt.show() - ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) - - num_col, peaks_neg_fin=self.find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) - - #num_col, peaks_neg_fin=find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) - x_min_hor_some=matrix_new[:,2][ (matrix_new[:,9]==0) ] - x_max_hor_some=matrix_new[:,3][ (matrix_new[:,9]==0) ] - cy_hor_some=matrix_new[:,5][ (matrix_new[:,9]==0) ] - arg_org_hor_some=matrix_new[:,0][ (matrix_new[:,9]==0) ] - - - - peaks_neg_tot=self.return_points_with_boundies(peaks_neg_fin,0, seperators_closeup_n[:,:,0].shape[1]) - - start_index_of_hor,newest_peaks,arg_min_hor_sort,lines_length_dels,lines_indexes_deleted=self.return_hor_spliter_by_index(peaks_neg_tot,x_min_hor_some,x_max_hor_some) - - arg_org_hor_some_sort=arg_org_hor_some[arg_min_hor_sort] - - - - start_index_of_hor_with_subset=[start_index_of_hor[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ]#start_index_of_hor[lines_length_dels>0] - arg_min_hor_sort_with_subset=[arg_min_hor_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - lines_indexes_deleted_with_subset=[lines_indexes_deleted[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - lines_length_dels_with_subset=[lines_length_dels[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - - arg_org_hor_some_sort_subset=[arg_org_hor_some_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - - #arg_min_hor_sort_with_subset=arg_min_hor_sort[lines_length_dels>0] - #lines_indexes_deleted_with_subset=lines_indexes_deleted[lines_length_dels>0] - #lines_length_dels_with_subset=lines_length_dels[lines_length_dels>0] - - - - - - - - vahid_subset=np.zeros((len(start_index_of_hor_with_subset),len(start_index_of_hor_with_subset)))-1 - for kkk1 in range(len(start_index_of_hor_with_subset)): - - - index_del_sub=np.unique(lines_indexes_deleted_with_subset[kkk1]) - - for kkk2 in range(len(start_index_of_hor_with_subset)): - - if set(lines_indexes_deleted_with_subset[kkk2][0]) < set(lines_indexes_deleted_with_subset[kkk1][0]): - vahid_subset[kkk1,kkk2]=kkk1 - else: - pass - #print(set(lines_indexes_deleted[kkk2][0]), set(lines_indexes_deleted[kkk1][0])) - - #print(vahid_subset,'zartt222') - - - # check the len of matrix if it has no length means that there is no spliter at all - - if len(vahid_subset>0): - #print('hihoo') - - - # find parenets args - line_int=np.zeros(vahid_subset.shape[0]) - - childs_id=[] - arg_child=[] - for li in range(vahid_subset.shape[0]): - #print(vahid_subset[:,li]) - if np.all(vahid_subset[:,li]==-1): - line_int[li]=-1 - else: - line_int[li]=1 - - - #childs_args_in=[ idd for idd in range(vahid_subset.shape[0]) if vahid_subset[idd,li]!=-1] - #helpi=[] - #for nad in range(len(childs_args_in)): - # helpi.append(arg_min_hor_sort_with_subset[childs_args_in[nad]]) - - - arg_child.append(arg_min_hor_sort_with_subset[li] ) - - - - - - #line_int=vahid_subset[0,:] - - #print(arg_child,line_int[0],'zartt33333') - arg_parent=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - start_index_of_hor_parent=[start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - #arg_parent=[lines_indexes_deleted_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - #arg_parent=[lines_length_dels_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - - - #arg_child=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] - start_index_of_hor_child=[start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] - - - - - cy_hor_some_sort=cy_hor_some[arg_parent] - - - - - - #print(start_index_of_hor, lines_length_dels ,lines_indexes_deleted,'zartt') - - #args_indexes=np.array(range(len(start_index_of_hor) )) - - newest_y_spliter_tot=[] - - for tj in range(len(newest_peaks)-1): - newest_y_spliter=[] - newest_y_spliter.append(spliter_y_new[i]) - if tj in np.unique(start_index_of_hor_parent): - ##print(cy_hor_some_sort) - cy_help=np.array(cy_hor_some_sort)[np.array(start_index_of_hor_parent)==tj] - cy_help_sort=np.sort(cy_help) - - #print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') - for mj in range(len(cy_help_sort)): - newest_y_spliter.append(cy_help_sort[mj]) - newest_y_spliter.append(spliter_y_new[i+1]) - - newest_y_spliter_tot.append(newest_y_spliter) - - - - - else: - line_int=[] - newest_y_spliter_tot=[] - - for tj in range(len(newest_peaks)-1): - newest_y_spliter=[] - newest_y_spliter.append(spliter_y_new[i]) - - newest_y_spliter.append(spliter_y_new[i+1]) - - newest_y_spliter_tot.append(newest_y_spliter) - - - - # if line_int is all -1 means that big spliters have no child and we can easily go through - if np.all(np.array(line_int)==-1): - for j in range(len(newest_peaks)-1): - newest_y_spliter=newest_y_spliter_tot[j] - - - for n in range(len(newest_y_spliter)-1): - #print(j,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'maaaa') - ##plt.imshow(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]]) - ##plt.show() - - #print(matrix_new[:,0][ (matrix_new[:,9]==1 )]) - for jvt in matrix_new[:,0][ (matrix_new[:,9]==1 ) & (matrix_new[:,6]> newest_y_spliter[n] ) & (matrix_new[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_new[:,1]) < newest_peaks[j+1] ) & (( matrix_new[:,1])> newest_peaks[j] ) ] : - pass - - ###plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) - #print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - num_col_sub, peaks_neg_fin_sub=self.find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.) - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - else: - for j in range(len(newest_peaks)-1): - newest_y_spliter=newest_y_spliter_tot[j] - - if j in start_index_of_hor_parent: - - x_min_ch=x_min_hor_some[arg_child] - x_max_ch=x_max_hor_some[arg_child] - cy_hor_some_sort_child=cy_hor_some[arg_child] - cy_hor_some_sort_child=np.sort(cy_hor_some_sort_child) - - #print(cy_hor_some_sort_child,'ychilds') - - for n in range(len(newest_y_spliter)-1): - - cy_child_in=cy_hor_some_sort_child[( cy_hor_some_sort_child>newest_y_spliter[n] ) & ( cy_hor_some_sort_child0: - num_col_ch, peaks_neg_ch=self.find_num_col( regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.0) - #print(peaks_neg_ch,'mizzzz') - #peaks_neg_ch=[] - #for djh in range(len(peaks_neg_ch)): - # peaks_neg_ch.append( peaks_neg_ch[djh]+newest_peaks[j] ) - - peaks_neg_ch_tot=self.return_points_with_boundies(peaks_neg_ch,newest_peaks[j], newest_peaks[j+1]) - - ss_in_ch,nst_p_ch,arg_n_ch,lines_l_del_ch,lines_in_del_ch=self.return_hor_spliter_by_index(peaks_neg_ch_tot,x_min_ch,x_max_ch) - - - - - - newest_y_spliter_ch_tot=[] - - for tjj in range(len(nst_p_ch)-1): - newest_y_spliter_new=[] - newest_y_spliter_new.append(newest_y_spliter[n]) - if tjj in np.unique(ss_in_ch): - - - #print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') - for mjj in range(len(cy_child_in)): - newest_y_spliter_new.append(cy_child_in[mjj]) - newest_y_spliter_new.append(newest_y_spliter[n+1]) - - newest_y_spliter_ch_tot.append(newest_y_spliter_new) - - - - - - for jn in range(len(nst_p_ch)-1): - newest_y_spliter_h=newest_y_spliter_ch_tot[jn] - - for nd in range(len(newest_y_spliter_h)-1): - - matrix_new_new2=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter_h[nd] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter_h[nd+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < nst_p_ch[jn+1] ) & (( matrix_of_lines_ch[:,1]-500)>nst_p_ch[jn] ) ] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if len( matrix_new_new2[:,9][matrix_new_new2[:,9]==1] )>0 and np.max(matrix_new_new2[:,8][matrix_new_new2[:,9]==1])>=0.2*(np.abs(newest_y_spliter_h[nd+1]-newest_y_spliter_h[nd] )): - num_col_sub_ch, peaks_neg_fin_sub_ch=self.find_num_col(regions_without_seperators[int(newest_y_spliter_h[nd]):int(newest_y_spliter_h[nd+1]),nst_p_ch[jn]:nst_p_ch[jn+1]],multiplier=5.0) - - else: - peaks_neg_fin_sub_ch=[] - - peaks_sub_ch=[] - peaks_sub_ch.append(nst_p_ch[jn]) - - for kjj in range(len(peaks_neg_fin_sub_ch)): - peaks_sub_ch.append(peaks_neg_fin_sub_ch[kjj]+nst_p_ch[jn]) - - peaks_sub_ch.append(nst_p_ch[jn+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for khh in range(len(peaks_sub_ch)-1): - boxes.append([ peaks_sub_ch[khh], peaks_sub_ch[khh+1] ,newest_y_spliter_h[nd],newest_y_spliter_h[nd+1]]) - - - - else: - - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - num_col_sub, peaks_neg_fin_sub=self.find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.0) - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - - - - else: - for n in range(len(newest_y_spliter)-1): - - - #plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) - #print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - num_col_sub, peaks_neg_fin_sub=self.find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.0) - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - - - - - - - else: - boxes.append([ 0, seperators_closeup_n[:,:,0].shape[1] ,spliter_y_new[i],spliter_y_new[i+1]]) - - - return boxes - - - - def return_boxes_of_images_by_order_of_reading_new(self,spliter_y_new,regions_without_seperators,matrix_of_lines_ch): - boxes=[] - - - # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ - # holes in the text and also finding spliter which covers more than one columns. - for i in range(len(spliter_y_new)-1): - #print(spliter_y_new[i],spliter_y_new[i+1]) - matrix_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,6]> spliter_y_new[i] ) & (matrix_of_lines_ch[:,7]< spliter_y_new[i+1] ) ] - #print(len( matrix_new[:,9][matrix_new[:,9]==1] )) - - #print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') - - # check to see is there any vertical seperator to find holes. - if 1>0:#len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): - - #org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) - #org_img_dichte=org_img_dichte-np.min(org_img_dichte) - ##plt.figure(figsize=(20,20)) - ##plt.plot(org_img_dichte) - ##plt.show() - ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) - - #print(int(spliter_y_new[i]),int(spliter_y_new[i+1]),'firssst') - - #plt.imshow(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:]) - #plt.show() - try: - num_col, peaks_neg_fin=self.find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.) - except: - peaks_neg_fin=[] - - #print(peaks_neg_fin,'peaks_neg_fin') - #num_col, peaks_neg_fin=find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) - x_min_hor_some=matrix_new[:,2][ (matrix_new[:,9]==0) ] - x_max_hor_some=matrix_new[:,3][ (matrix_new[:,9]==0) ] - cy_hor_some=matrix_new[:,5][ (matrix_new[:,9]==0) ] - arg_org_hor_some=matrix_new[:,0][ (matrix_new[:,9]==0) ] - - - peaks_neg_tot=self.return_points_with_boundies(peaks_neg_fin,0, regions_without_seperators[:,:].shape[1]) - - start_index_of_hor,newest_peaks,arg_min_hor_sort,lines_length_dels,lines_indexes_deleted=self.return_hor_spliter_by_index_for_without_verticals(peaks_neg_tot,x_min_hor_some,x_max_hor_some) - - arg_org_hor_some_sort=arg_org_hor_some[arg_min_hor_sort] - - - start_index_of_hor_with_subset=[start_index_of_hor[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ]#start_index_of_hor[lines_length_dels>0] - arg_min_hor_sort_with_subset=[arg_min_hor_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - lines_indexes_deleted_with_subset=[lines_indexes_deleted[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - lines_length_dels_with_subset=[lines_length_dels[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - - arg_org_hor_some_sort_subset=[arg_org_hor_some_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij]>0 ] - - #arg_min_hor_sort_with_subset=arg_min_hor_sort[lines_length_dels>0] - #lines_indexes_deleted_with_subset=lines_indexes_deleted[lines_length_dels>0] - #lines_length_dels_with_subset=lines_length_dels[lines_length_dels>0] - - - - - - - - vahid_subset=np.zeros((len(start_index_of_hor_with_subset),len(start_index_of_hor_with_subset)))-1 - for kkk1 in range(len(start_index_of_hor_with_subset)): - - - index_del_sub=np.unique(lines_indexes_deleted_with_subset[kkk1]) - - for kkk2 in range(len(start_index_of_hor_with_subset)): - - if set(lines_indexes_deleted_with_subset[kkk2][0]) < set(lines_indexes_deleted_with_subset[kkk1][0]): - vahid_subset[kkk1,kkk2]=kkk1 - else: - pass - #print(set(lines_indexes_deleted[kkk2][0]), set(lines_indexes_deleted[kkk1][0])) - - - - # check the len of matrix if it has no length means that there is no spliter at all - - if len(vahid_subset>0): - #print('hihoo') - - - # find parenets args - line_int=np.zeros(vahid_subset.shape[0]) - - childs_id=[] - arg_child=[] - for li in range(vahid_subset.shape[0]): - #print(vahid_subset[:,li]) - if np.all(vahid_subset[:,li]==-1): - line_int[li]=-1 - else: - line_int[li]=1 - - - #childs_args_in=[ idd for idd in range(vahid_subset.shape[0]) if vahid_subset[idd,li]!=-1] - #helpi=[] - #for nad in range(len(childs_args_in)): - # helpi.append(arg_min_hor_sort_with_subset[childs_args_in[nad]]) - - - arg_child.append(arg_min_hor_sort_with_subset[li] ) - - - - - - #line_int=vahid_subset[0,:] - - - arg_parent=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - start_index_of_hor_parent=[start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - #arg_parent=[lines_indexes_deleted_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - #arg_parent=[lines_length_dels_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] - - - #arg_child=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] - start_index_of_hor_child=[start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] - - - - - cy_hor_some_sort=cy_hor_some[arg_parent] - - - - - - #print(start_index_of_hor, lines_length_dels ,lines_indexes_deleted,'zartt') - - #args_indexes=np.array(range(len(start_index_of_hor) )) - - newest_y_spliter_tot=[] - - for tj in range(len(newest_peaks)-1): - newest_y_spliter=[] - newest_y_spliter.append(spliter_y_new[i]) - if tj in np.unique(start_index_of_hor_parent): - #print(cy_hor_some_sort) - cy_help=np.array(cy_hor_some_sort)[np.array(start_index_of_hor_parent)==tj] - cy_help_sort=np.sort(cy_help) - - #print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') - for mj in range(len(cy_help_sort)): - newest_y_spliter.append(cy_help_sort[mj]) - newest_y_spliter.append(spliter_y_new[i+1]) - - newest_y_spliter_tot.append(newest_y_spliter) - - - - - else: - line_int=[] - newest_y_spliter_tot=[] - - for tj in range(len(newest_peaks)-1): - newest_y_spliter=[] - newest_y_spliter.append(spliter_y_new[i]) - - newest_y_spliter.append(spliter_y_new[i+1]) - - newest_y_spliter_tot.append(newest_y_spliter) - - - - # if line_int is all -1 means that big spliters have no child and we can easily go through - if np.all(np.array(line_int)==-1): - for j in range(len(newest_peaks)-1): - newest_y_spliter=newest_y_spliter_tot[j] - - - for n in range(len(newest_y_spliter)-1): - #print(j,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'maaaa') - ##plt.imshow(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]]) - ##plt.show() - - #print(matrix_new[:,0][ (matrix_new[:,9]==1 )]) - for jvt in matrix_new[:,0][ (matrix_new[:,9]==1 ) & (matrix_new[:,6]> newest_y_spliter[n] ) & (matrix_new[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_new[:,1]) < newest_peaks[j+1] ) & (( matrix_new[:,1])> newest_peaks[j] ) ] : - pass - - ###plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) - #print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - #print( int(newest_y_spliter[n]),int(newest_y_spliter[n+1]),newest_peaks[j],newest_peaks[j+1] ) - try: - num_col_sub, peaks_neg_fin_sub=self.find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=7.) - except: - peaks_neg_fin_sub=[] - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - else: - for j in range(len(newest_peaks)-1): - - - newest_y_spliter=newest_y_spliter_tot[j] - - if j in start_index_of_hor_parent: - - x_min_ch=x_min_hor_some[arg_child] - x_max_ch=x_max_hor_some[arg_child] - cy_hor_some_sort_child=cy_hor_some[arg_child] - cy_hor_some_sort_child=np.sort(cy_hor_some_sort_child) - - - - for n in range(len(newest_y_spliter)-1): - - cy_child_in=cy_hor_some_sort_child[( cy_hor_some_sort_child>newest_y_spliter[n] ) & ( cy_hor_some_sort_child0: - try: - num_col_ch, peaks_neg_ch=self.find_num_col( regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=7.0) - except: - peaks_neg_ch=[] - #print(peaks_neg_ch,'mizzzz') - #peaks_neg_ch=[] - #for djh in range(len(peaks_neg_ch)): - # peaks_neg_ch.append( peaks_neg_ch[djh]+newest_peaks[j] ) - - peaks_neg_ch_tot=self.return_points_with_boundies(peaks_neg_ch,newest_peaks[j], newest_peaks[j+1]) - - - ss_in_ch,nst_p_ch,arg_n_ch,lines_l_del_ch,lines_in_del_ch=self.return_hor_spliter_by_index_for_without_verticals(peaks_neg_ch_tot,x_min_ch,x_max_ch) - - - - - - newest_y_spliter_ch_tot=[] - - for tjj in range(len(nst_p_ch)-1): - newest_y_spliter_new=[] - newest_y_spliter_new.append(newest_y_spliter[n]) - if tjj in np.unique(ss_in_ch): - - - #print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') - for mjj in range(len(cy_child_in)): - newest_y_spliter_new.append(cy_child_in[mjj]) - newest_y_spliter_new.append(newest_y_spliter[n+1]) - - newest_y_spliter_ch_tot.append(newest_y_spliter_new) - - - - - - for jn in range(len(nst_p_ch)-1): - newest_y_spliter_h=newest_y_spliter_ch_tot[jn] - - for nd in range(len(newest_y_spliter_h)-1): - - matrix_new_new2=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter_h[nd] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter_h[nd+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < nst_p_ch[jn+1] ) & (( matrix_of_lines_ch[:,1]-500)>nst_p_ch[jn] ) ] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new2[:,9][matrix_new_new2[:,9]==1] )>0 and np.max(matrix_new_new2[:,8][matrix_new_new2[:,9]==1])>=0.2*(np.abs(newest_y_spliter_h[nd+1]-newest_y_spliter_h[nd] )): - try: - num_col_sub_ch, peaks_neg_fin_sub_ch=self.find_num_col(regions_without_seperators[int(newest_y_spliter_h[nd]):int(newest_y_spliter_h[nd+1]),nst_p_ch[jn]:nst_p_ch[jn+1]],multiplier=7.0) - except: - peaks_neg_fin_sub_ch=[] - - else: - peaks_neg_fin_sub_ch=[] - - peaks_sub_ch=[] - peaks_sub_ch.append(nst_p_ch[jn]) - - for kjj in range(len(peaks_neg_fin_sub_ch)): - peaks_sub_ch.append(peaks_neg_fin_sub_ch[kjj]+nst_p_ch[jn]) - - peaks_sub_ch.append(nst_p_ch[jn+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for khh in range(len(peaks_sub_ch)-1): - boxes.append([ peaks_sub_ch[khh], peaks_sub_ch[khh+1] ,newest_y_spliter_h[nd],newest_y_spliter_h[nd+1]]) - - - - else: - - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - try: - num_col_sub, peaks_neg_fin_sub=self.find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=7.0) - except: - peaks_neg_fin_sub=[] - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - - - - else: - for n in range(len(newest_y_spliter)-1): - - - #plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) - #print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) - matrix_new_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,9]==1 ) & (matrix_of_lines_ch[:,6]> newest_y_spliter[n] ) & (matrix_of_lines_ch[:,7]< newest_y_spliter[n+1] ) & ( ( matrix_of_lines_ch[:,1]+500) < newest_peaks[j+1] ) & (( matrix_of_lines_ch[:,1]-500)> newest_peaks[j] )] - #print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') - if 1>0:#len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): - try: - num_col_sub, peaks_neg_fin_sub=self.find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.0) - except: - peaks_neg_fin_sub=[] - else: - peaks_neg_fin_sub=[] - - peaks_sub=[] - peaks_sub.append(newest_peaks[j]) - - for kj in range(len(peaks_neg_fin_sub)): - peaks_sub.append(peaks_neg_fin_sub[kj]+newest_peaks[j]) - - peaks_sub.append(newest_peaks[j+1]) - - #peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) - - for kh in range(len(peaks_sub)-1): - boxes.append([ peaks_sub[kh], peaks_sub[kh+1] ,newest_y_spliter[n],newest_y_spliter[n+1]]) - - - - - - - - - else: - boxes.append([ 0, regions_without_seperators[:,:].shape[1] ,spliter_y_new[i],spliter_y_new[i+1]]) - - - return boxes - - def return_boxes_of_images_by_order_of_reading_2cols(self,spliter_y_new,regions_without_seperators,matrix_of_lines_ch,seperators_closeup_n): - boxes=[] - - - # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ - # holes in the text and also finding spliter which covers more than one columns. - for i in range(len(spliter_y_new)-1): - #print(spliter_y_new[i],spliter_y_new[i+1]) - matrix_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,6]> spliter_y_new[i] ) & (matrix_of_lines_ch[:,7]< spliter_y_new[i+1] ) ] - #print(len( matrix_new[:,9][matrix_new[:,9]==1] )) - - #print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') - - # check to see is there any vertical seperator to find holes. - if 1>0:#len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): - #print(int(spliter_y_new[i]),int(spliter_y_new[i+1]),'burayaaaa galimiirrrrrrrrrrrrrrrrrrrrrrrrrrr') - #org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) - #org_img_dichte=org_img_dichte-np.min(org_img_dichte) - ##plt.figure(figsize=(20,20)) - ##plt.plot(org_img_dichte) - ##plt.show() - ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) - - try: - num_col, peaks_neg_fin=self.find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) - - - except: - peaks_neg_fin=[] - num_col=0 - - peaks_neg_tot=self.return_points_with_boundies(peaks_neg_fin,0, seperators_closeup_n[:,:,0].shape[1]) - - for kh in range(len(peaks_neg_tot)-1): - boxes.append([ peaks_neg_tot[kh], peaks_neg_tot[kh+1] ,spliter_y_new[i],spliter_y_new[i+1]]) - - else: - boxes.append([ 0, seperators_closeup_n[:,:,0].shape[1] ,spliter_y_new[i],spliter_y_new[i+1]]) - - - - return boxes - def return_hor_spliter_by_index(self,peaks_neg_fin_t,x_min_hor_some,x_max_hor_some): - - arg_min_hor_sort=np.argsort(x_min_hor_some) - x_min_hor_some_sort=np.sort(x_min_hor_some) - x_max_hor_some_sort=x_max_hor_some[arg_min_hor_sort] - - arg_minmax=np.array(range(len(peaks_neg_fin_t))) - indexer_lines=[] - indexes_to_delete=[] - indexer_lines_deletions_len=[] - indexr_uniq_ind=[] - for i in range(len(x_min_hor_some_sort)): - min_h=peaks_neg_fin_t-x_min_hor_some_sort[i] - max_h=peaks_neg_fin_t-x_max_hor_some_sort[i] - - min_h[0]=min_h[0]#+20 - max_h[len(max_h)-1]=max_h[len(max_h)-1]##-20 - - min_h_neg=arg_minmax[(min_h<0) & (np.abs(min_h)<360) ] - max_h_neg=arg_minmax[(max_h>=0) & (np.abs(max_h)<360) ] - - if len(min_h_neg)>0 and len(max_h_neg)>0: - deletions=list(range(min_h_neg[0]+1,max_h_neg[0])) - unique_delets_int=[] - #print(deletions,len(deletions),'delii') - if len(deletions)>0: - #print(deletions,len(deletions),'delii2') - - for j in range(len(deletions)): - indexes_to_delete.append(deletions[j]) - #print(deletions,indexes_to_delete,'badiii') - unique_delets=np.unique(indexes_to_delete) - #print(min_h_neg[0],unique_delets) - unique_delets_int=unique_delets[unique_delets0 and min_h_neg[0]>=2: - index_line_true=index_line_true - else: - index_line_true=min_h_neg[0] - - indexer_lines.append(index_line_true) - - if len(unique_delets_int)>0: - for dd in range(len(unique_delets_int)): - indexes_to_delete.append(unique_delets_int[dd]) - else: - indexer_lines.append(-999) - indexer_lines_deletions_len.append(-999) - indexr_uniq_ind.append(-999) - - peaks_true=[] - for m in range(len(peaks_neg_fin_t)): - if m in indexes_to_delete: - pass - else: - peaks_true.append(peaks_neg_fin_t[m]) - return indexer_lines,peaks_true,arg_min_hor_sort,indexer_lines_deletions_len,indexr_uniq_ind - def return_region_segmentation_after_implementing_not_head_maintext_parallel(self,image_regions_eraly_p,boxes): - image_revised=np.zeros((image_regions_eraly_p.shape[0] , image_regions_eraly_p.shape[1])) - for i in range(len(boxes)): - - image_box=image_regions_eraly_p[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1])] - image_box=np.array(image_box) - #plt.imshow(image_box) - #plt.show() - - #print(int(boxes[i][2]),int(boxes[i][3]),int(boxes[i][0]),int(boxes[i][1]),'addaa') - image_box=self.implent_law_head_main_not_parallel(image_box) - image_box=self.implent_law_head_main_not_parallel(image_box) - image_box=self.implent_law_head_main_not_parallel(image_box) - - image_revised[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1])]=image_box[:,:] - return image_revised - - def tear_main_texts_on_the_boundaries_of_boxes(self,img_revised_tab,boxes): - for i in range(len(boxes)): - img_revised_tab[ int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][1]-10):int(boxes[i][1]), 0][img_revised_tab[ int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][1]-10):int(boxes[i][1]), 0]==1]=0 - img_revised_tab[ int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][1]-10):int(boxes[i][1]), 1][img_revised_tab[ int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][1]-10):int(boxes[i][1]), 1]==1]=0 - img_revised_tab[ int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][1]-10):int(boxes[i][1]), 2][img_revised_tab[ int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][1]-10):int(boxes[i][1]), 2]==1]=0 - return img_revised_tab - - - def implent_law_head_main_not_parallel(self,text_regions): - #print(text_regions.shape) - text_indexes=[1 , 2 ]# 1: main text , 2: header , 3: comments - - - for t_i in text_indexes: - textline_mask=(text_regions[:,:]==t_i) - textline_mask=textline_mask*255.0 - - textline_mask=textline_mask.astype(np.uint8) - textline_mask=np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) - kernel = np.ones((5,5),np.uint8) - - #print(type(textline_mask),np.unique(textline_mask),textline_mask.shape) - imgray = cv2.cvtColor(textline_mask, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - if t_i==1: - contours_main,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - #print(type(contours_main)) - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - #print(contours_main[0],np.shape(contours_main[0]),contours_main[0][:,0,0]) - elif t_i==2: - contours_header,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - #print(type(contours_header)) - areas_header=np.array([cv2.contourArea(contours_header[j]) for j in range(len(contours_header))]) - M_header=[cv2.moments(contours_header[j]) for j in range(len(contours_header))] - cx_header=[(M_header[j]['m10']/(M_header[j]['m00']+1e-32)) for j in range(len(M_header))] - cy_header=[(M_header[j]['m01']/(M_header[j]['m00']+1e-32)) for j in range(len(M_header))] - - x_min_header=np.array([np.min(contours_header[j][:,0,0]) for j in range(len(contours_header))]) - x_max_header=np.array([np.max(contours_header[j][:,0,0]) for j in range(len(contours_header))]) - - y_min_header=np.array([np.min(contours_header[j][:,0,1]) for j in range(len(contours_header))]) - y_max_header=np.array([np.max(contours_header[j][:,0,1]) for j in range(len(contours_header))]) - - args=np.array(range(1,len(cy_header)+1)) - args_main=np.array(range(1,len(cy_main)+1)) - for jj in range(len(contours_main)): - headers_in_main=[(cy_header>y_min_main[jj]) & ((cy_headery_min_main[jj]) & ((cy_main0] - res_true=res-1 - - args_log_main=args_main*mains_in_main - res_main=args_log_main[args_log_main>0] - res_true_main=res_main-1 - - if len(res_true)>0: - sum_header=np.sum(areas_header[res_true]) - sum_main=np.sum(areas_main[res_true_main]) - if sum_main>sum_header: - cnt_int=[contours_header[j] for j in res_true] - text_regions=cv2.fillPoly(text_regions, pts =cnt_int, color=(1,1,1)) - else: - cnt_int=[contours_main[j] for j in res_true_main] - text_regions=cv2.fillPoly(text_regions, pts =cnt_int, color=(2,2,2)) - - for jj in range(len(contours_header)): - main_in_header=[(cy_main>y_min_header[jj]) & ((cy_mainy_min_header[jj]) & ((cy_header0] - res_true=res-1 - - args_log_header=args*header_in_header - res_header=args_log_header[args_log_header>0] - res_true_header=res_header-1 - - if len(res_true)>0: - - sum_header=np.sum(areas_header[res_true_header]) - sum_main=np.sum(areas_main[res_true]) - - if sum_main>sum_header: - - cnt_int=[contours_header[j] for j in res_true_header] - text_regions=cv2.fillPoly(text_regions, pts =cnt_int, color=(1,1,1)) - else: - cnt_int=[contours_main[j] for j in res_true] - text_regions=cv2.fillPoly(text_regions, pts =cnt_int, color=(2,2,2)) - - - - - return text_regions - - def delete_seperator_around(self,spliter_y,peaks_neg,image_by_region): - # format of subboxes box=[x1, x2 , y1, y2] - - if len(image_by_region.shape)==3: - for i in range(len(spliter_y)-1): - for j in range(1,len(peaks_neg[i])-1): - image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0]==6 ]=0 - image_by_region[spliter_y[i]:spliter_y[i+1],peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),1]==6 ]=0 - image_by_region[spliter_y[i]:spliter_y[i+1],peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),2]==6 ]=0 - - image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0]==7 ]=0 - image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),1]==7 ]=0 - image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j]),2]==7 ]=0 - else: - for i in range(len(spliter_y)-1): - for j in range(1,len(peaks_neg[i])-1): - image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j])][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j])]==6 ]=0 - - image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j])][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-int(1./20.*peaks_neg[i][j]):peaks_neg[i][j]+int(1./20.*peaks_neg[i][j])]==7 ]=0 - return image_by_region - - def find_features_of_contoures(self,contours_main): - - - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - - - return y_min_main ,y_max_main - - - def add_tables_heuristic_to_layout(self,image_regions_eraly_p,boxes,slope_mean_hor,spliter_y,peaks_neg_tot,image_revised): - - image_revised_1=self.delete_seperator_around(spliter_y,peaks_neg_tot,image_revised) - img_comm_e=np.zeros(image_revised_1.shape) - img_comm=np.repeat(img_comm_e[:, :, np.newaxis], 3, axis=2) - - for indiv in np.unique(image_revised_1): - - #print(indiv,'indd') - image_col=(image_revised_1==indiv)*255 - img_comm_in=np.repeat(image_col[:, :, np.newaxis], 3, axis=2) - img_comm_in=img_comm_in.astype(np.uint8) - - imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY) - - - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - - main_contours=self.filter_contours_area_of_image_tables(thresh,contours,hirarchy,max_area=1,min_area=0.0001) - - - img_comm=cv2.fillPoly(img_comm, pts =main_contours, color=(indiv,indiv,indiv)) - ###img_comm_in=cv2.fillPoly(img_comm, pts =interior_contours, color=(0,0,0)) - - - #img_comm=np.repeat(img_comm[:, :, np.newaxis], 3, axis=2) - img_comm=img_comm.astype(np.uint8) - - - - - - if not self.isNaN(slope_mean_hor): - image_revised_last=np.zeros((image_regions_eraly_p.shape[0] , image_regions_eraly_p.shape[1],3)) - for i in range(len(boxes)): - - - image_box=img_comm[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:] - - image_box_tabels_1=(image_box[:,:,0]==7)*1 - - - - - - contours_tab,_=self.return_contours_of_image(image_box_tabels_1) - - contours_tab=self.filter_contours_area_of_image_tables(image_box_tabels_1,contours_tab,_,1,0.001) - - image_box_tabels_1=(image_box[:,:,0]==6)*1 - - - image_box_tabels_and_m_text=( (image_box[:,:,0]==7) | (image_box[:,:,0]==1) )*1 - image_box_tabels_and_m_text=image_box_tabels_and_m_text.astype(np.uint8) - - - image_box_tabels_1=image_box_tabels_1.astype(np.uint8) - image_box_tabels_1 = cv2.dilate(image_box_tabels_1,self.kernel,iterations = 5) - - - - - contours_table_m_text,_=self.return_contours_of_image(image_box_tabels_and_m_text) - - - image_box_tabels=np.repeat(image_box_tabels_1[:, :, np.newaxis], 3, axis=2) - - image_box_tabels=image_box_tabels.astype(np.uint8) - imgray = cv2.cvtColor(image_box_tabels, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - - contours_line,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - - y_min_main_line ,y_max_main_line=self.find_features_of_contoures(contours_line) - #_,_,y_min_main_line ,y_max_main_line,x_min_main_line,x_max_main_line=find_new_features_of_contoures(contours_line) - y_min_main_tab ,y_max_main_tab=self.find_features_of_contoures(contours_tab) - - cx_tab_m_text,cy_tab_m_text ,x_min_tab_m_text , x_max_tab_m_text, y_min_tab_m_text ,y_max_tab_m_text=self.find_new_features_of_contoures(contours_table_m_text) - cx_tabl,cy_tabl ,x_min_tabl , x_max_tabl, y_min_tabl ,y_max_tabl,_=self.find_new_features_of_contoures(contours_tab) - - - if len(y_min_main_tab )>0: - y_down_tabs=[] - y_up_tabs=[] - - for i_t in range(len(y_min_main_tab )): - y_down_tab=[] - y_up_tab=[] - for i_l in range(len(y_min_main_line)): - if y_min_main_tab[i_t]>y_min_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l] and y_min_main_tab[i_t]>y_max_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l]: - pass - elif y_min_main_tab[i_t]= x_min_tab_m_text[v_n] and cy_tabl[i_t]<= y_max_tab_m_text[v_n] and cy_tabl[i_t]>= y_min_tab_m_text[v_n] and cx_tabl[i_t]!=cx_tab_m_text[v_n] and cy_tabl[i_t]!=cy_tab_m_text[v_n]: - y_up_tabs.append(y_min_tab_m_text[v_n]) - y_down_tabs.append(y_max_tab_m_text[v_n]) - #y_up_tabs.append(y_min_main_tab[i_t]) - #y_down_tabs.append(y_max_main_tab[i_t]) - else: - y_up_tabs.append(np.min(y_up_tab)) - y_down_tabs.append(np.max(y_down_tab)) - - - else: - y_down_tabs=[] - y_up_tabs=[] - pass - - for ii in range(len(y_up_tabs)): - image_box[y_up_tabs[ii]:y_down_tabs[ii],:,0]=7 - - - - - - - image_revised_last[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:]=image_box[:,:,:] - - - else: - for i in range(len(boxes)): - - image_box=img_comm[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:] - image_revised_last[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:]=image_box[:,:,:] - - ##plt.figure(figsize=(20,20)) - ##plt.imshow(image_box[:,:,0]) - ##plt.show() - return image_revised_last - def find_features_of_contours(self,contours_main): - - - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - - - - return y_min_main ,y_max_main,areas_main - def remove_headers_and_mains_intersection(self,seperators_closeup_n,img_revised_tab,boxes): - for ind in range(len(boxes)): - asp=np.zeros((img_revised_tab[:,:,0].shape[0],seperators_closeup_n[:,:,0].shape[1])) - asp[ int(boxes[ind][2]):int(boxes[ind][3]),int(boxes[ind][0]):int(boxes[ind][1])] = img_revised_tab[ int(boxes[ind][2]):int(boxes[ind][3]),int(boxes[ind][0]):int(boxes[ind][1]),0] - - head_patch_con=( asp[:,:]==2)*1 - main_patch_con=( asp[:,:]==1)*1 - #print(head_patch_con) - head_patch_con=head_patch_con.astype(np.uint8) - main_patch_con=main_patch_con.astype(np.uint8) - - head_patch_con=np.repeat(head_patch_con[:, :, np.newaxis], 3, axis=2) - main_patch_con=np.repeat(main_patch_con[:, :, np.newaxis], 3, axis=2) - - imgray = cv2.cvtColor(head_patch_con, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - - contours_head_patch_con,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - contours_head_patch_con=self.return_parent_contours(contours_head_patch_con,hiearchy) - - imgray = cv2.cvtColor(main_patch_con, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - - contours_main_patch_con,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - contours_main_patch_con=self.return_parent_contours(contours_main_patch_con,hiearchy) - - - y_patch_head_min, y_patch_head_max,_= self.find_features_of_contours(contours_head_patch_con) - y_patch_main_min, y_patch_main_max,_= self.find_features_of_contours(contours_main_patch_con) - - - for i in range(len(y_patch_head_min)): - for j in range(len(y_patch_main_min)): - if y_patch_head_max[i]>y_patch_main_min[j] and y_patch_head_min[i]np.sum(areas_main): - img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0][img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0]==1 ]=2 - else: - img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0][img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0]==2 ]=1 - - - - elif y_patch_head_min[i]y_patch_main_max[j]: - y_down=y_patch_main_max[j] - y_up=y_patch_head_min[i] - - patch_intersection=np.zeros(asp.shape) - patch_intersection[y_up:y_down,:]=asp[y_up:y_down,:] - - - head_patch_con=( patch_intersection[:,:]==2)*1 - main_patch_con=( patch_intersection[:,:]==1)*1 - head_patch_con=head_patch_con.astype(np.uint8) - main_patch_con=main_patch_con.astype(np.uint8) - - head_patch_con=np.repeat(head_patch_con[:, :, np.newaxis], 3, axis=2) - main_patch_con=np.repeat(main_patch_con[:, :, np.newaxis], 3, axis=2) - - imgray = cv2.cvtColor(head_patch_con, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - - contours_head_patch_con,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - contours_head_patch_con=self.return_parent_contours(contours_head_patch_con,hiearchy) - - imgray = cv2.cvtColor(main_patch_con, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - - contours_main_patch_con,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - contours_main_patch_con=self.return_parent_contours(contours_main_patch_con,hiearchy) - - _,_,areas_head= self.find_features_of_contours(contours_head_patch_con) - _,_,areas_main= self.find_features_of_contours(contours_main_patch_con) - - if np.sum(areas_head)>np.sum(areas_main): - img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0][img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0]==1 ]=2 - else: - img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0][img_revised_tab[y_up:y_down,int(boxes[ind][0]):int(boxes[ind][1]),0]==2 ]=1 - - - - #print(np.unique(patch_intersection) ) - ##plt.figure(figsize=(20,20)) - ##plt.imshow(patch_intersection) - ##plt.show() - else: - pass - - return img_revised_tab - - def order_of_regions(self,textline_mask,contours_main,contours_header, y_ref): - - ##plt.imshow(textline_mask) - ##plt.show() - """ - print(len(contours_main),'contours_main') - mada_n=textline_mask.sum(axis=1) - y=mada_n[:] - - y_help=np.zeros(len(y)+40) - y_help[20:len(y)+20]=y - x=np.array( range(len(y)) ) - - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - - ##plt.imshow(textline_mask[:,:]) - ##plt.show() - - - sigma_gaus=8 - - z= gaussian_filter1d(y_help, sigma_gaus) - zneg_rev=-y_help+np.max(y_help) - - zneg=np.zeros(len(zneg_rev)+40) - zneg[20:len(zneg_rev)+20]=zneg_rev - zneg= gaussian_filter1d(zneg, sigma_gaus) - - - peaks, _ = find_peaks(z, height=0) - peaks_neg, _ = find_peaks(zneg, height=0) - - peaks_neg=peaks_neg-20-20 - peaks=peaks-20 - """ - - textline_sum_along_width=textline_mask.sum(axis=1) - - y=textline_sum_along_width[:] - y_padded=np.zeros(len(y)+40) - y_padded[20:len(y)+20]=y - x=np.array( range(len(y)) ) - - - peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - - - sigma_gaus=8 - - z= gaussian_filter1d(y_padded, sigma_gaus) - zneg_rev=-y_padded+np.max(y_padded) - - zneg=np.zeros(len(zneg_rev)+40) - zneg[20:len(zneg_rev)+20]=zneg_rev - zneg= gaussian_filter1d(zneg, sigma_gaus) - - - peaks, _ = find_peaks(z, height=0) - peaks_neg, _ = find_peaks(zneg, height=0) - - peaks_neg=peaks_neg-20-20 - peaks=peaks-20 - - - ##plt.plot(z) - ##plt.show() - - if contours_main!=None: - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - - if len(contours_header)!=None: - areas_header=np.array([cv2.contourArea(contours_header[j]) for j in range(len(contours_header))]) - M_header=[cv2.moments(contours_header[j]) for j in range(len(contours_header))] - cx_header=[(M_header[j]['m10']/(M_header[j]['m00']+1e-32)) for j in range(len(M_header))] - cy_header=[(M_header[j]['m01']/(M_header[j]['m00']+1e-32)) for j in range(len(M_header))] - - x_min_header=np.array([np.min(contours_header[j][:,0,0]) for j in range(len(contours_header))]) - x_max_header=np.array([np.max(contours_header[j][:,0,0]) for j in range(len(contours_header))]) - - y_min_header=np.array([np.min(contours_header[j][:,0,1]) for j in range(len(contours_header))]) - y_max_header=np.array([np.max(contours_header[j][:,0,1]) for j in range(len(contours_header))]) - #print(cy_main,'mainy') - - peaks_neg_new=[] - - peaks_neg_new.append(0+y_ref) - for iii in range(len(peaks_neg)): - peaks_neg_new.append(peaks_neg[iii]+y_ref) - - peaks_neg_new.append(textline_mask.shape[0]+y_ref) - - - if len(cy_main)>0 and np.max(cy_main)>np.max(peaks_neg_new): - cy_main=np.array(cy_main)*(np.max(peaks_neg_new)/np.max(cy_main) )-10 - - - - - if contours_main!=None: - indexer_main=np.array(range(len(contours_main))) - - - if contours_main!=None: - len_main=len(contours_main) - else: - len_main=0 - - - matrix_of_orders=np.zeros((len(contours_main)+len(contours_header),5)) - - matrix_of_orders[:,0]=np.array( range( len(contours_main)+len(contours_header) ) ) - - matrix_of_orders[:len(contours_main),1]=1 - matrix_of_orders[len(contours_main):,1]=2 - - matrix_of_orders[:len(contours_main),2]=cx_main - matrix_of_orders[len(contours_main):,2]=cx_header - - matrix_of_orders[:len(contours_main),3]=cy_main - matrix_of_orders[len(contours_main):,3]=cy_header - - - matrix_of_orders[:len(contours_main),4]=np.array( range( len(contours_main) ) ) - matrix_of_orders[len(contours_main):,4]=np.array( range( len(contours_header) ) ) - - - - - #print(peaks_neg_new,'peaks_neg_new') - - #print(matrix_of_orders,'matrix_of_orders') - #print(peaks_neg_new,np.max(peaks_neg_new)) - final_indexers_sorted=[] - final_types=[] - final_index_type=[] - for i in range(len(peaks_neg_new)-1): - top=peaks_neg_new[i] - down=peaks_neg_new[i+1] - - #print(top,down,'topdown') - - indexes_in=matrix_of_orders[:,0][(matrix_of_orders[:,3]>=top) & ((matrix_of_orders[:,3]=top) & ((matrix_of_orders[:,3]=top) & ((matrix_of_orders[:,3]=top) & ((matrix_of_orders[:,3]=top) & ((matrix_of_orders[:,3]0: - order_of_texts.append(interest[0]) - index_b+=1 - else: - pass - - for mm in range(len(found_polygons_text_region_h)): - id_of_texts.append('r'+str(index_b) ) - interest=indexes_sorted_2[index_of_types_2[mm]] - order_of_texts.append(interest) - index_b+=1 - - return order_of_texts, id_of_texts - - - def get_text_region_boxes_by_given_contours(self,contours): - - - kernel = np.ones((5,5),np.uint8) - boxes=[] - contours_new=[] - for jj in range(len(contours)): - x,y,w,h = cv2.boundingRect(contours[jj]) - - boxes.append([x,y,w,h]) - contours_new.append(contours[jj]) - - del contours - return boxes,contours_new - - - def return_teilwiese_deskewed_lines(self,text_regions_p,textline_rotated): - - kernel = np.ones((5, 5), np.uint8) - textline_rotated = cv2.erode(textline_rotated, kernel, iterations=1) - - textline_rotated_new=np.zeros(textline_rotated.shape) - rgb_m=1 - rgb_h=2 - - cnt_m,boxes_m=self.return_contours_of_interested_region_and_bounding_box(text_regions_p,rgb_m) - cnt_h,boxes_h=self.return_contours_of_interested_region_and_bounding_box(text_regions_p,rgb_h) - - areas_cnt_m=np.array([cv2.contourArea(cnt_m[j]) for j in range(len(cnt_m))]) - - argmax=np.argmax(areas_cnt_m) - - #plt.imshow(textline_rotated[ boxes_m[argmax][1]:boxes_m[argmax][1]+boxes_m[argmax][3] ,boxes_m[argmax][0]:boxes_m[argmax][0]+boxes_m[argmax][2]]) - #plt.show() - - - - for argmax in range(len(boxes_m)): - - textline_text_region=textline_rotated[ boxes_m[argmax][1]:boxes_m[argmax][1]+boxes_m[argmax][3] ,boxes_m[argmax][0]:boxes_m[argmax][0]+boxes_m[argmax][2] ] - - textline_text_region_revised=self.seperate_lines_new(textline_text_region,0) - #except: - # textline_text_region_revised=textline_rotated[ boxes_m[argmax][1]:boxes_m[argmax][1]+boxes_m[argmax][3] ,boxes_m[argmax][0]:boxes_m[argmax][0]+boxes_m[argmax][2] ] - textline_rotated_new[boxes_m[argmax][1]:boxes_m[argmax][1]+boxes_m[argmax][3] ,boxes_m[argmax][0]:boxes_m[argmax][0]+boxes_m[argmax][2]]=textline_text_region_revised[:,:] - - #textline_rotated_new[textline_rotated_new>0]=1 - #textline_rotated_new[textline_rotated_new<0]=0 - #plt.imshow(textline_rotated_new) - #plt.show() - def return_contours_of_interested_region_and_bounding_box(self,region_pre_p,pixel): - - # pixels of images are identified by 5 - cnts_images=(region_pre_p[:,:,0]==pixel)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=1,min_area=0.0003) - - boxes = [] - - for jj in range(len(contours_imgs)): - x, y, w, h = cv2.boundingRect(contours_imgs[jj]) - boxes.append([int(x), int(y), int(w), int(h)]) - return contours_imgs,boxes - def find_number_of_columns_in_document(self,region_pre_p,num_col_classifier,pixel_lines,contours_h=None): - - seperators_closeup=( (region_pre_p[:,:,:]==pixel_lines))*1 - - seperators_closeup[0:110,:,:]=0 - seperators_closeup[seperators_closeup.shape[0]-150:,:,:]=0 - - kernel = np.ones((5,5),np.uint8) - - seperators_closeup=seperators_closeup.astype(np.uint8) - seperators_closeup = cv2.dilate(seperators_closeup,kernel,iterations = 1) - seperators_closeup = cv2.erode(seperators_closeup,kernel,iterations = 1) - - ##plt.imshow(seperators_closeup[:,:,0]) - ##plt.show() - seperators_closeup_new=np.zeros((seperators_closeup.shape[0] ,seperators_closeup.shape[1] )) - - - - ##_,seperators_closeup_n=self.combine_hor_lines_and_delete_cross_points_and_get_lines_features_back(region_pre_p[:,:,0]) - seperators_closeup_n=np.copy(seperators_closeup) - - seperators_closeup_n=seperators_closeup_n.astype(np.uint8) - ##plt.imshow(seperators_closeup_n[:,:,0]) - ##plt.show() - - seperators_closeup_n_binary=np.zeros(( seperators_closeup_n.shape[0],seperators_closeup_n.shape[1]) ) - seperators_closeup_n_binary[:,:]=seperators_closeup_n[:,:,0] - - seperators_closeup_n_binary[:,:][seperators_closeup_n_binary[:,:]!=0]=1 - #seperators_closeup_n_binary[:,:][seperators_closeup_n_binary[:,:]==0]=255 - #seperators_closeup_n_binary[:,:][seperators_closeup_n_binary[:,:]==-255]=0 - - - #seperators_closeup_n_binary=(seperators_closeup_n_binary[:,:]==2)*1 - - #gray = cv2.cvtColor(seperators_closeup_n, cv2.COLOR_BGR2GRAY) - - #print(np.unique(seperators_closeup_n_binary)) - - ##plt.imshow(seperators_closeup_n_binary) - ##plt.show() - - - #print( np.unique(gray),np.unique(seperators_closeup_n[:,:,1]) ) - - gray = cv2.bitwise_not(seperators_closeup_n_binary) - gray=gray.astype(np.uint8) - - - ##plt.imshow(gray) - ##plt.show() - bw = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, \ - cv2.THRESH_BINARY, 15, -2) - ##plt.imshow(bw[:,:]) - ##plt.show() - - horizontal = np.copy(bw) - vertical = np.copy(bw) - - cols = horizontal.shape[1] - horizontal_size = cols // 30 - # Create structure element for extracting horizontal lines through morphology operations - horizontalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontal_size, 1)) - # Apply morphology operations - horizontal = cv2.erode(horizontal, horizontalStructure) - horizontal = cv2.dilate(horizontal, horizontalStructure) - - kernel = np.ones((5,5),np.uint8) - - - horizontal = cv2.dilate(horizontal,kernel,iterations = 2) - horizontal = cv2.erode(horizontal,kernel,iterations = 2) - #plt.imshow(horizontal) - #plt.show() - - rows = vertical.shape[0] - verticalsize = rows // 30 - # Create structure element for extracting vertical lines through morphology operations - verticalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (1, verticalsize)) - # Apply morphology operations - vertical = cv2.erode(vertical, verticalStructure) - vertical = cv2.dilate(vertical, verticalStructure) - - vertical = cv2.dilate(vertical,kernel,iterations = 1) - # Show extracted vertical lines - - horizontal,special_seperators=self.combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new(vertical,horizontal) - - - ##plt.imshow(vertical) - ##plt.show() - #print(vertical.shape,np.unique(vertical),'verticalvertical') - seperators_closeup_new[:,:][vertical[:,:]!=0]=1 - seperators_closeup_new[:,:][horizontal[:,:]!=0]=1 - - ##plt.imshow(seperators_closeup_new) - ##plt.show() - ##seperators_closeup_n - vertical=np.repeat(vertical[:, :, np.newaxis], 3, axis=2) - vertical=vertical.astype(np.uint8) - - ##plt.plot(vertical[:,:,0].sum(axis=0)) - ##plt.show() - - #plt.plot(vertical[:,:,0].sum(axis=1)) - #plt.show() - - imgray = cv2.cvtColor(vertical, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_line_vers,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=self.find_features_of_lines(contours_line_vers) - #print(slope_lines,'vertical') - args=np.array( range(len(slope_lines) )) - args_ver=args[slope_lines==1] - dist_x_ver=dist_x[slope_lines==1] - y_min_main_ver=y_min_main[slope_lines==1] - y_max_main_ver=y_max_main[slope_lines==1] - x_min_main_ver=x_min_main[slope_lines==1] - x_max_main_ver=x_max_main[slope_lines==1] - cx_main_ver=cx_main[slope_lines==1] - dist_y_ver=y_max_main_ver-y_min_main_ver - len_y=seperators_closeup.shape[0]/3.0 - - - #plt.imshow(horizontal) - #plt.show() - - horizontal=np.repeat(horizontal[:, :, np.newaxis], 3, axis=2) - horizontal=horizontal.astype(np.uint8) - imgray = cv2.cvtColor(horizontal, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_line_hors,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=self.find_features_of_lines(contours_line_hors) - - slope_lines_org_hor=slope_lines_org[slope_lines==0] - args=np.array( range(len(slope_lines) )) - len_x=seperators_closeup.shape[1]/5.0 - - dist_y=np.abs(y_max_main-y_min_main) - - args_hor=args[slope_lines==0] - dist_x_hor=dist_x[slope_lines==0] - y_min_main_hor=y_min_main[slope_lines==0] - y_max_main_hor=y_max_main[slope_lines==0] - x_min_main_hor=x_min_main[slope_lines==0] - x_max_main_hor=x_max_main[slope_lines==0] - dist_y_hor=dist_y[slope_lines==0] - cy_main_hor=cy_main[slope_lines==0] - - args_hor=args_hor[dist_x_hor>=len_x/2.0] - x_max_main_hor=x_max_main_hor[dist_x_hor>=len_x/2.0] - x_min_main_hor=x_min_main_hor[dist_x_hor>=len_x/2.0] - cy_main_hor=cy_main_hor[dist_x_hor>=len_x/2.0] - y_min_main_hor=y_min_main_hor[dist_x_hor>=len_x/2.0] - y_max_main_hor=y_max_main_hor[dist_x_hor>=len_x/2.0] - dist_y_hor=dist_y_hor[dist_x_hor>=len_x/2.0] - - slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x/2.0] - dist_x_hor=dist_x_hor[dist_x_hor>=len_x/2.0] - - - matrix_of_lines_ch=np.zeros((len(cy_main_hor)+len(cx_main_ver),10)) - - matrix_of_lines_ch[:len(cy_main_hor),0]=args_hor - matrix_of_lines_ch[len(cy_main_hor):,0]=args_ver - - - matrix_of_lines_ch[len(cy_main_hor):,1]=cx_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),2]=x_min_main_hor+50#x_min_main_hor+150 - matrix_of_lines_ch[len(cy_main_hor):,2]=x_min_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),3]=x_max_main_hor-50#x_max_main_hor-150 - matrix_of_lines_ch[len(cy_main_hor):,3]=x_max_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),4]=dist_x_hor - matrix_of_lines_ch[len(cy_main_hor):,4]=dist_x_ver - - matrix_of_lines_ch[:len(cy_main_hor),5]=cy_main_hor - - - matrix_of_lines_ch[:len(cy_main_hor),6]=y_min_main_hor - matrix_of_lines_ch[len(cy_main_hor):,6]=y_min_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),7]=y_max_main_hor - matrix_of_lines_ch[len(cy_main_hor):,7]=y_max_main_ver - - matrix_of_lines_ch[:len(cy_main_hor),8]=dist_y_hor - matrix_of_lines_ch[len(cy_main_hor):,8]=dist_y_ver - - - matrix_of_lines_ch[len(cy_main_hor):,9]=1 - - - - if contours_h is not None: - slope_lines_head,dist_x_head, x_min_main_head ,x_max_main_head ,cy_main_head,slope_lines_org_head,y_min_main_head, y_max_main_head, cx_main_head=self.find_features_of_lines(contours_h) - matrix_l_n=np.zeros((matrix_of_lines_ch.shape[0]+len(cy_main_head),matrix_of_lines_ch.shape[1])) - matrix_l_n[:matrix_of_lines_ch.shape[0],:]=np.copy(matrix_of_lines_ch[:,:]) - args_head=np.array(range(len(cy_main_head)))+len(cy_main_hor) - - matrix_l_n[matrix_of_lines_ch.shape[0]:,0]=args_head - matrix_l_n[matrix_of_lines_ch.shape[0]:,2]=x_min_main_head+30 - matrix_l_n[matrix_of_lines_ch.shape[0]:,3]=x_max_main_head-30 - - matrix_l_n[matrix_of_lines_ch.shape[0]:,4]=dist_x_head - - matrix_l_n[matrix_of_lines_ch.shape[0]:,5]=y_min_main_head-3-8 - matrix_l_n[matrix_of_lines_ch.shape[0]:,6]=y_min_main_head-5-8 - matrix_l_n[matrix_of_lines_ch.shape[0]:,7]=y_min_main_head+1-8 - matrix_l_n[matrix_of_lines_ch.shape[0]:,8]=4 - - matrix_of_lines_ch=np.copy(matrix_l_n) - - #print(matrix_of_lines_ch) - - - - - - - """ - - - - seperators_closeup=seperators_closeup.astype(np.uint8) - imgray = cv2.cvtColor(seperators_closeup, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_lines,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=self.find_features_of_lines(contours_lines) - - slope_lines_org_hor=slope_lines_org[slope_lines==0] - args=np.array( range(len(slope_lines) )) - len_x=seperators_closeup.shape[1]/4.0 - - args_hor=args[slope_lines==0] - dist_x_hor=dist_x[slope_lines==0] - x_min_main_hor=x_min_main[slope_lines==0] - x_max_main_hor=x_max_main[slope_lines==0] - cy_main_hor=cy_main[slope_lines==0] - - args_hor=args_hor[dist_x_hor>=len_x/2.0] - x_max_main_hor=x_max_main_hor[dist_x_hor>=len_x/2.0] - x_min_main_hor=x_min_main_hor[dist_x_hor>=len_x/2.0] - cy_main_hor=cy_main_hor[dist_x_hor>=len_x/2.0] - slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x/2.0] - - - slope_lines_org_hor=slope_lines_org_hor[np.abs(slope_lines_org_hor)<1.2] - slope_mean_hor=np.mean(slope_lines_org_hor) - - - - args_ver=args[slope_lines==1] - y_min_main_ver=y_min_main[slope_lines==1] - y_max_main_ver=y_max_main[slope_lines==1] - x_min_main_ver=x_min_main[slope_lines==1] - x_max_main_ver=x_max_main[slope_lines==1] - cx_main_ver=cx_main[slope_lines==1] - dist_y_ver=y_max_main_ver-y_min_main_ver - len_y=seperators_closeup.shape[0]/3.0 - - - - print(matrix_of_lines_ch[:,8][matrix_of_lines_ch[:,9]==0],'khatlarrrr') - args_main_spliters=matrix_of_lines_ch[:,0][ (matrix_of_lines_ch[:,9]==0) & ((matrix_of_lines_ch[:,8]<=290)) & ((matrix_of_lines_ch[:,2]<=.16*region_pre_p.shape[1])) & ((matrix_of_lines_ch[:,3]>=.84*region_pre_p.shape[1]))] - - cy_main_spliters=matrix_of_lines_ch[:,5][ (matrix_of_lines_ch[:,9]==0) & ((matrix_of_lines_ch[:,8]<=290)) & ((matrix_of_lines_ch[:,2]<=.16*region_pre_p.shape[1])) & ((matrix_of_lines_ch[:,3]>=.84*region_pre_p.shape[1]))] - """ - - cy_main_spliters=cy_main_hor[ (x_min_main_hor<=.16*region_pre_p.shape[1]) & (x_max_main_hor>=.84*region_pre_p.shape[1] )] - - cy_main_spliters=np.array( list(cy_main_spliters)+list(special_seperators)) - - if contours_h is not None: - try: - cy_main_spliters_head=cy_main_head[ (x_min_main_head<=.16*region_pre_p.shape[1]) & (x_max_main_head>=.84*region_pre_p.shape[1] )] - cy_main_spliters=np.array( list(cy_main_spliters)+list(cy_main_spliters_head)) - except: - pass - args_cy_spliter=np.argsort(cy_main_spliters) - - cy_main_spliters_sort=cy_main_spliters[args_cy_spliter] - - spliter_y_new=[] - spliter_y_new.append(0) - for i in range(len(cy_main_spliters_sort)): - spliter_y_new.append( cy_main_spliters_sort[i] ) - - spliter_y_new.append(region_pre_p.shape[0]) - - spliter_y_new_diff=np.diff(spliter_y_new)/float(region_pre_p.shape[0])*100 - - args_big_parts=np.array(range(len(spliter_y_new_diff))) [ spliter_y_new_diff>22 ] - - - - regions_without_seperators=self.return_regions_without_seperators(region_pre_p) - - ##print(args_big_parts,'args_big_parts') - #image_page_otsu=self.otsu_copy(image_page_deskewd) - #print(np.unique(image_page_otsu[:,:,0])) - #image_page_background_zero=self.image_change_background_pixels_to_zero(image_page_otsu) - - - length_y_threshold=regions_without_seperators.shape[0]/4.0 - - num_col_fin=0 - peaks_neg_fin_fin=[] - - for iteils in args_big_parts: - - - regions_without_seperators_teil=regions_without_seperators[int(spliter_y_new[iteils]):int(spliter_y_new[iteils+1]),:,0] - #image_page_background_zero_teil=image_page_background_zero[int(spliter_y_new[iteils]):int(spliter_y_new[iteils+1]),:] - - #print(regions_without_seperators_teil.shape) - ##plt.imshow(regions_without_seperators_teil) - ##plt.show() - - #num_col, peaks_neg_fin=self.find_num_col(regions_without_seperators_teil,multiplier=6.0) - - #regions_without_seperators_teil=cv2.erode(regions_without_seperators_teil,kernel,iterations = 3) - # - num_col, peaks_neg_fin=self.find_num_col(regions_without_seperators_teil,multiplier=7.0) - - if num_col>num_col_fin: - num_col_fin=num_col - peaks_neg_fin_fin=peaks_neg_fin - """ - #print(length_y_vertical_lines,length_y_threshold,'x_center_of_ver_linesx_center_of_ver_linesx_center_of_ver_lines') - if len(cx_main_ver)>0 and len( dist_y_ver[dist_y_ver>=length_y_threshold] ) >=1: - num_col, peaks_neg_fin=self.find_num_col(regions_without_seperators_teil,multiplier=6.0) - else: - #plt.imshow(image_page_background_zero_teil) - #plt.show() - #num_col, peaks_neg_fin=self.find_num_col_only_image(image_page_background_zero,multiplier=2.4)#2.3) - num_col, peaks_neg_fin=self.find_num_col_only_image(image_page_background_zero_teil,multiplier=3.4)#2.3) - - print(num_col,'birda') - if num_col>0: - pass - elif num_col==0: - print(num_col,'birda2222') - num_col_regions, peaks_neg_fin_regions=self.find_num_col(regions_without_seperators_teil,multiplier=10.0) - if num_col_regions==0: - pass - else: - - num_col=num_col_regions - peaks_neg_fin=peaks_neg_fin_regions[:] - """ - - #print(num_col+1,'num colmsssssssss') - - if len(args_big_parts)==1 and (len(peaks_neg_fin_fin)+1)=500] - peaks_neg_fin=peaks_neg_fin[peaks_neg_fin<=(vertical.shape[1]-500)] - peaks_neg_fin_fin=peaks_neg_fin[:] - - #print(peaks_neg_fin_fin,'peaks_neg_fin_fintaza') - - - return num_col_fin, peaks_neg_fin_fin,matrix_of_lines_ch,spliter_y_new,seperators_closeup_n - - - - def return_contours_of_interested_region_by_size(self,region_pre_p,pixel,min_area,max_area): - - # pixels of images are identified by 5 - if len(region_pre_p.shape)==3: - cnts_images=(region_pre_p[:,:,0]==pixel)*1 - else: - cnts_images=(region_pre_p[:,:]==pixel)*1 - cnts_images=cnts_images.astype(np.uint8) - cnts_images=np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs,hiearchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs=self.return_parent_contours(contours_imgs,hiearchy) - contours_imgs=self.filter_contours_area_of_image_tables(thresh,contours_imgs,hiearchy,max_area=max_area,min_area=min_area) - - img_ret=np.zeros((region_pre_p.shape[0],region_pre_p.shape[1],3)) - img_ret=cv2.fillPoly(img_ret,pts=contours_imgs, color=(1,1,1)) - return img_ret[:,:,0] - - - def get_regions_from_xy_neu(self,img): - img_org=np.copy(img) - - img_height_h=img_org.shape[0] - img_width_h=img_org.shape[1] - - model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p) - - gaussian_filter=False - patches=True - binary=True - - - - - ratio_x=1 - ratio_y=1 - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_org=self.do_prediction(patches,img,model_region) - - prediction_regions_org=self.resize_image(prediction_regions_org, img_height_h, img_width_h ) - - #plt.imshow(prediction_regions_org[:,:,0]) - #plt.show() - #sys.exit() - prediction_regions_org=prediction_regions_org[:,:,0] - - - - - gaussian_filter=False - patches=False - binary=False - - - - - ratio_x=1 - ratio_y=1 - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_orgt=self.do_prediction(patches,img,model_region) - - prediction_regions_orgt=self.resize_image(prediction_regions_orgt, img_height_h, img_width_h ) - - #plt.imshow(prediction_regions_orgt[:,:,0]) - #plt.show() - #sys.exit() - prediction_regions_orgt=prediction_regions_orgt[:,:,0] - - - mask_texts_longshot=(prediction_regions_orgt[:,:]==1)*1 - - mask_texts_longshot=np.uint8(mask_texts_longshot) - #mask_texts_longshot = cv2.dilate(mask_texts_longshot[:,:], self.kernel, iterations=2) - - pixel_img=1 - polygons_of_only_texts_longshot=self.return_contours_of_interested_region(mask_texts_longshot,pixel_img) - - longshot_true=np.zeros(mask_texts_longshot.shape) - #text_regions_p_true[:,:]=text_regions_p_1[:,:] - - longshot_true=cv2.fillPoly(longshot_true,pts=polygons_of_only_texts_longshot, color=(1,1,1)) - - #plt.imshow(longshot_true) - #plt.show() - - gaussian_filter=False - patches=False - binary=False - - - - - ratio_x=1 - ratio_y=1 - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - one_third_upper_ny=int(img.shape[0]/3.0) - - img=img[0:one_third_upper_ny,:,:] - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_longshot_one_third=self.do_prediction(patches,img,model_region) - - prediction_regions_longshot_one_third=self.resize_image(prediction_regions_longshot_one_third, one_third_upper_ny, img_width_h ) - - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - img=img[one_third_upper_ny:int(2*one_third_upper_ny),:,:] - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_longshot_one_third_middle=self.do_prediction(patches,img,model_region) - - prediction_regions_longshot_one_third_middle=self.resize_image(prediction_regions_longshot_one_third_middle, one_third_upper_ny, img_width_h ) - - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - img=img[int(2*one_third_upper_ny):,:,:] - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_longshot_one_third_down=self.do_prediction(patches,img,model_region) - - prediction_regions_longshot_one_third_down=self.resize_image(prediction_regions_longshot_one_third_down, img_height_h-int(2*one_third_upper_ny), img_width_h ) - - #plt.imshow(prediction_regions_org[:,:,0]) - #plt.show() - #sys.exit() - prediction_regions_longshot=np.zeros((img_height_h,img_width_h)) - - #prediction_regions_longshot=prediction_regions_longshot[:,:,0] - - #prediction_regions_longshot[0:one_third_upper_ny,:]=prediction_regions_longshot_one_third[:,:,0] - #prediction_regions_longshot[one_third_upper_ny:int(2*one_third_upper_ny):,:]=prediction_regions_longshot_one_third_middle[:,:,0] - #prediction_regions_longshot[int(2*one_third_upper_ny):,:]=prediction_regions_longshot_one_third_down[:,:,0] - - prediction_regions_longshot=longshot_true[:,:] - #plt.imshow(prediction_regions_longshot) - #plt.show() - - gaussian_filter=False - patches=True - binary=False - - - - - ratio_x=1#1.1 - ratio_y=1 - median_blur=False - - #img= self.resize_image(img_org, int(img_org.shape[0]*0.8), int(img_org.shape[1]*1.6)) - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - - prediction_regions=self.do_prediction(patches,img,model_region) - text_region1=self.resize_image(prediction_regions, img_height_h, img_width_h ) - - #plt.imshow(text_region1[:,:,0]) - #plt.show() - ratio_x=1 - ratio_y=1.2#1.3 - binary=False - median_blur=False - - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - - prediction_regions=self.do_prediction(patches,img,model_region) - text_region2=self.resize_image(prediction_regions, img_height_h, img_width_h ) - - - #plt.imshow(text_region2[:,:,0]) - #plt.show() - session_region.close() - del model_region - del session_region - gc.collect() - - #text_region1=text_region1[:,:,0] - #text_region2=text_region2[:,:,0] - - #text_region1[(text_region1[:,:]==2) & (text_region2[:,:]==1)]=1 - - - mask_zeros_from_1=(text_region2[:,:,0]==0)*1 - #mask_text_from_1=(text_region1[:,:,0]==1)*1 - - - - mask_img_text_region1=(text_region1[:,:,0]==2)*1 - text_region2_1st_channel=text_region1[:,:,0] - - text_region2_1st_channel[mask_zeros_from_1==1]=0 - - ##text_region2_1st_channel[mask_img_text_region1[:,:]==1]=2 - #text_region2_1st_channel[(mask_text_from_1==1) & (text_region2_1st_channel==2)]=1 - - mask_lines1=(text_region1[:,:,0]==3)*1 - mask_lines2=(text_region2[:,:,0]==3)*1 - - mask_lines2[mask_lines1[:,:]==1]=1 - - #plt.imshow(text_region2_1st_channel) - #plt.show() - - text_region2_1st_channel = cv2.erode(text_region2_1st_channel[:,:], self.kernel, iterations=4) - - #plt.imshow(text_region2_1st_channel) - #plt.show() - - text_region2_1st_channel = cv2.dilate(text_region2_1st_channel[:,:], self.kernel, iterations=4) - - - text_region2_1st_channel[mask_lines2[:,:]==1]=3 - - #text_region2_1st_channel[ (prediction_regions_org[:,:]==1) & (text_region2_1st_channel[:,:]==2)]=1 - - - # only in the case of model 3 - - text_region2_1st_channel[(prediction_regions_longshot[:,:]==1) & (text_region2_1st_channel[:,:]==2)]=1 - - text_region2_1st_channel[ (prediction_regions_org[:,:]==2) & (text_region2_1st_channel[:,:]==0)]=2 - - #text_region2_1st_channel[prediction_regions_org[:,:]==0]=0 - - #plt.imshow(text_region2_1st_channel) - #plt.show() - - #text_region2_1st_channel[:,:400]=0 - - mask_texts_only=(text_region2_1st_channel[:,:]==1)*1 - - mask_images_only=(text_region2_1st_channel[:,:]==2)*1 - - mask_lines_only=(text_region2_1st_channel[:,:]==3)*1 - - pixel_img=1 - polygons_of_only_texts=self.return_contours_of_interested_region(mask_texts_only,pixel_img) - - polygons_of_only_images=self.return_contours_of_interested_region(mask_images_only,pixel_img) - - polygons_of_only_lines=self.return_contours_of_interested_region(mask_lines_only,pixel_img) - - - text_regions_p_true=np.zeros(text_region2_1st_channel.shape) - #text_regions_p_true[:,:]=text_regions_p_1[:,:] - - text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_lines, color=(3,3,3)) - - text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_images, color=(2,2,2)) - - text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) - - ##print(np.unique(text_regions_p_true)) - - - #text_regions_p_true_3d=np.repeat(text_regions_p_1[:, :, np.newaxis], 3, axis=2) - #text_regions_p_true_3d=text_regions_p_true_3d.astype(np.uint8) - - - return text_regions_p_true#text_region2_1st_channel - - def get_regions_from_xy(self,img): - img_org=np.copy(img) - - img_height_h=img_org.shape[0] - img_width_h=img_org.shape[1] - - model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p) - - gaussian_filter=False - patches=True - binary=True - - - - - ratio_x=1 - ratio_y=1 - median_blur=False - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_org=self.do_prediction(patches,img,model_region) - - ###plt.imshow(prediction_regions_org[:,:,0]) - ###plt.show() - ##sys.exit() - prediction_regions_org=prediction_regions_org[:,:,0] - - - - gaussian_filter=False - patches=True - binary=False - - - - - ratio_x=1.1 - ratio_y=1 - median_blur=False - - #img= self.resize_image(img_org, int(img_org.shape[0]*0.8), int(img_org.shape[1]*1.6)) - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - - prediction_regions=self.do_prediction(patches,img,model_region) - text_region1=self.resize_image(prediction_regions, img_height_h, img_width_h ) - - - ratio_x=1 - ratio_y=1.1 - binary=False - median_blur=False - - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - - prediction_regions=self.do_prediction(patches,img,model_region) - text_region2=self.resize_image(prediction_regions, img_height_h, img_width_h ) - - session_region.close() - del model_region - del session_region - gc.collect() - - - mask_zeros_from_1=(text_region1[:,:,0]==0)*1 - #mask_text_from_1=(text_region1[:,:,0]==1)*1 - - - - mask_img_text_region1=(text_region1[:,:,0]==2)*1 - text_region2_1st_channel=text_region2[:,:,0] - - text_region2_1st_channel[mask_zeros_from_1==1]=0 - - text_region2_1st_channel[mask_img_text_region1[:,:]==1]=2 - #text_region2_1st_channel[(mask_text_from_1==1) & (text_region2_1st_channel==2)]=1 - - mask_lines1=(text_region1[:,:,0]==3)*1 - mask_lines2=(text_region2[:,:,0]==3)*1 - - mask_lines2[mask_lines1[:,:]==1]=1 - - ##plt.imshow(text_region2_1st_channel) - ##plt.show() - - text_region2_1st_channel = cv2.erode(text_region2_1st_channel[:,:], self.kernel, iterations=5) - - ##plt.imshow(text_region2_1st_channel) - ##plt.show() - - text_region2_1st_channel = cv2.dilate(text_region2_1st_channel[:,:], self.kernel, iterations=5) - - - text_region2_1st_channel[mask_lines2[:,:]==1]=3 - - text_region2_1st_channel[ (prediction_regions_org[:,:]==1) & (text_region2_1st_channel[:,:]==2)]=1 - text_region2_1st_channel[prediction_regions_org[:,:]==3]=3 - - ##plt.imshow(text_region2_1st_channel) - ##plt.show() - return text_region2_1st_channel - - - def rotation_image_new(self,img,thetha): - rotated=imutils.rotate(img,thetha) - return self.rotate_max_area_new(img, rotated,thetha) - - def rotate_max_area_new(self,image,rotated,angle): - wr, hr =self.rotatedRectWithMaxArea(image.shape[1], image.shape[0],math.radians(angle)) - h, w, _ = rotated.shape - y1 = h//2 - int(hr/2) - y2 = y1 + int(hr) - x1 = w//2 - int(wr/2) - x2 = x1 + int(wr) - return rotated[y1:y2, x1:x2] - - - def rotation_not_90_func(self,img,textline,text_regions_p_1,thetha): - rotated=imutils.rotate(img,thetha) - rotated_textline=imutils.rotate(textline,thetha) - rotated_layout=imutils.rotate(text_regions_p_1,thetha) - return self.rotate_max_area(img, rotated,rotated_textline,rotated_layout,thetha) - - def rotate_max_area(self,image,rotated,rotated_textline,rotated_layout,angle): - wr, hr =self.rotatedRectWithMaxArea(image.shape[1], image.shape[0],math.radians(angle)) - h, w, _ = rotated.shape - y1 = h//2 - int(hr/2) - y2 = y1 + int(hr) - x1 = w//2 - int(wr/2) - x2 = x1 + int(wr) - return rotated[y1:y2, x1:x2],rotated_textline[y1:y2, x1:x2],rotated_layout[y1:y2, x1:x2] - - - def rotation_not_90_func_full_layout(self,img,textline,text_regions_p_1,text_regions_p_fully,thetha): - rotated=imutils.rotate(img,thetha) - rotated_textline=imutils.rotate(textline,thetha) - rotated_layout=imutils.rotate(text_regions_p_1,thetha) - rotated_layout_full=imutils.rotate(text_regions_p_fully,thetha) - return self.rotate_max_area_full_layout(img, rotated,rotated_textline,rotated_layout,rotated_layout_full,thetha) - - def rotate_max_area_full_layout(self,image,rotated,rotated_textline,rotated_layout,rotated_layout_full,angle): - wr, hr =self.rotatedRectWithMaxArea(image.shape[1], image.shape[0],math.radians(angle)) - h, w, _ = rotated.shape - y1 = h//2 - int(hr/2) - y2 = y1 + int(hr) - x1 = w//2 - int(wr/2) - x2 = x1 + int(wr) - return rotated[y1:y2, x1:x2],rotated_textline[y1:y2, x1:x2],rotated_layout[y1:y2, x1:x2],rotated_layout_full[y1:y2, x1:x2] - - def get_regions_from_xy_2models_ens(self,img): - img_org=np.copy(img) - - img_height_h=img_org.shape[0] - img_width_h=img_org.shape[1] - - model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) - - - gaussian_filter=False - patches=False - binary=False - - ratio_x=1 - ratio_y=1 - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - prediction_regions_long=self.do_prediction(patches,img,model_region) - - prediction_regions_long=self.resize_image(prediction_regions_long, img_height_h, img_width_h ) - - - gaussian_filter=False - patches=True - binary=False - - - - - ratio_x=1 - ratio_y=1.2 - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_org_y=self.do_prediction(patches,img,model_region) - - prediction_regions_org_y=self.resize_image(prediction_regions_org_y, img_height_h, img_width_h ) - - #plt.imshow(prediction_regions_org[:,:,0]) - #plt.show() - #sys.exit() - prediction_regions_org_y=prediction_regions_org_y[:,:,0] - - - mask_zeros_y=(prediction_regions_org_y[:,:]==0)*1 - - - - - - - ratio_x=1.2 - ratio_y=1 - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_org=self.do_prediction(patches,img,model_region) - - prediction_regions_org=self.resize_image(prediction_regions_org, img_height_h, img_width_h ) - - #plt.imshow(prediction_regions_org[:,:,0]) - #plt.show() - #sys.exit() - prediction_regions_org=prediction_regions_org[:,:,0] - - prediction_regions_org[(prediction_regions_org[:,:]==1) & (mask_zeros_y[:,:]==1)]=0 - - - prediction_regions_org[( prediction_regions_long[:,:,0]==1 ) & (prediction_regions_org[:,:]==2) ]=1 - - session_region.close() - del model_region - del session_region - gc.collect() - - - - - return prediction_regions_org - - def get_regions_from_xy_2models(self,img,is_image_enhanced): - img_org=np.copy(img) - - img_height_h=img_org.shape[0] - img_width_h=img_org.shape[1] - - model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) - - gaussian_filter=False - patches=True - binary=False - - - - - - ratio_y=1.3 - ratio_x=1 - - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - - if median_blur: - img=cv2.medianBlur(img,5) - if gaussian_filter: - img= cv2.GaussianBlur(img,(5,5),0) - img = img.astype(np.uint16) - prediction_regions_org_y=self.do_prediction(patches,img,model_region) - - prediction_regions_org_y=self.resize_image(prediction_regions_org_y, img_height_h, img_width_h ) - - #plt.imshow(prediction_regions_org_y[:,:,0]) - #plt.show() - #sys.exit() - prediction_regions_org_y=prediction_regions_org_y[:,:,0] - - - mask_zeros_y=(prediction_regions_org_y[:,:]==0)*1 - - - - - - if is_image_enhanced: - ratio_x=1.2 - else: - ratio_x=1 - - ratio_y=1 - median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) - img = img.astype(np.uint16) - if median_blur: img=cv2.medianBlur(img,5) if gaussian_filter: img= cv2.GaussianBlur(img,(5,5),0) img = img.astype(np.uint16) - prediction_regions_org=self.do_prediction(patches,img,model_region) - - prediction_regions_org=self.resize_image(prediction_regions_org, img_height_h, img_width_h ) - + prediction_regions_org=self.do_prediction(patches,img,model_region) + + prediction_regions_org=resize_image(prediction_regions_org, img_height_h, img_width_h ) + ##plt.imshow(prediction_regions_org[:,:,0]) ##plt.show() ##sys.exit() prediction_regions_org=prediction_regions_org[:,:,0] - + prediction_regions_org[(prediction_regions_org[:,:]==1) & (mask_zeros_y[:,:]==1)]=0 session_region.close() del model_region del session_region gc.collect() ###K.clear_session() - + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p2) - + gaussian_filter=False patches=True binary=False - - - + + + ratio_x=1 ratio_y=1 median_blur=False - - img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - + + img= resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) + if binary: - img = self.otsu_copy_binary(img)#self.otsu_copy(img) + img = otsu_copy_binary(img)#self.otsu_copy(img) img = img.astype(np.uint16) - + if median_blur: img=cv2.medianBlur(img,5) if gaussian_filter: img= cv2.GaussianBlur(img,(5,5),0) img = img.astype(np.uint16) - + marginal_patch=0.2 prediction_regions_org2=self.do_prediction(patches,img,model_region,marginal_patch) - - prediction_regions_org2=self.resize_image(prediction_regions_org2, img_height_h, img_width_h ) - + + prediction_regions_org2=resize_image(prediction_regions_org2, img_height_h, img_width_h ) + #plt.imshow(prediction_regions_org2[:,:,0]) #plt.show() #sys.exit() ##prediction_regions_org=prediction_regions_org[:,:,0] - + session_region.close() del model_region del session_region gc.collect() ###K.clear_session() - + mask_zeros2=(prediction_regions_org2[:,:,0]==0)*1 mask_lines2=(prediction_regions_org2[:,:,0]==3)*1 - + text_sume_early=( (prediction_regions_org[:,:]==1)*1 ).sum() - - + + prediction_regions_org_copy=np.copy(prediction_regions_org) - + prediction_regions_org_copy[(prediction_regions_org_copy[:,:]==1) & (mask_zeros2[:,:]==1)]=0 - + text_sume_second=( (prediction_regions_org_copy[:,:]==1)*1 ).sum() - + rate_two_models=text_sume_second/float(text_sume_early)*100 - + print(rate_two_models,'ratio_of_two_models') if is_image_enhanced and rate_two_models<95.50:#98.45: pass else: prediction_regions_org=np.copy(prediction_regions_org_copy) - + ##prediction_regions_org[mask_lines2[:,:]==1]=3 prediction_regions_org[(mask_lines2[:,:]==1) & (prediction_regions_org[:,:]==0)]=3 - - + + del mask_lines2 del mask_zeros2 del prediction_regions_org2 - + #if is_image_enhanced: #pass #else: #model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p2) - + #gaussian_filter=False #patches=True #binary=False - - - + + + #ratio_x=1 #ratio_y=1 #median_blur=False - - #img= self.resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - + + #img= resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) + #if binary: #img = self.otsu_copy_binary(img)#self.otsu_copy(img) #img = img.astype(np.uint16) - + #if median_blur: #img=cv2.medianBlur(img,5) #if gaussian_filter: #img= cv2.GaussianBlur(img,(5,5),0) #img = img.astype(np.uint16) #prediction_regions_org2=self.do_prediction(patches,img,model_region) - - #prediction_regions_org2=self.resize_image(prediction_regions_org2, img_height_h, img_width_h ) - + + #prediction_regions_org2=resize_image(prediction_regions_org2, img_height_h, img_width_h ) + ##plt.imshow(prediction_regions_org2[:,:,0]) ##plt.show() ##sys.exit() ###prediction_regions_org=prediction_regions_org[:,:,0] - + #session_region.close() #del model_region #del session_region #gc.collect() ####K.clear_session() - + #mask_zeros2=(prediction_regions_org2[:,:,0]==0)*1 #mask_lines2=(prediction_regions_org2[:,:,0]==3)*1 - + #text_sume_early=( (prediction_regions_org[:,:]==1)*1 ).sum() - + #prediction_regions_org[(prediction_regions_org[:,:]==1) & (mask_zeros2[:,:]==1)]=0 - + ###prediction_regions_org[mask_lines2[:,:]==1]=3 #prediction_regions_org[(mask_lines2[:,:]==1) & (prediction_regions_org[:,:]==0)]=3 - + #text_sume_second=( (prediction_regions_org[:,:]==1)*1 ).sum() - + #print(text_sume_second/float(text_sume_early)*100,'twomodelsratio') - + #del mask_lines2 #del mask_zeros2 #del prediction_regions_org2 - + mask_lines_only=(prediction_regions_org[:,:]==3)*1 - + prediction_regions_org = cv2.erode(prediction_regions_org[:,:], self.kernel, iterations=2) - + #plt.imshow(text_region2_1st_channel) #plt.show() - + prediction_regions_org = cv2.dilate(prediction_regions_org[:,:], self.kernel, iterations=2) - + mask_texts_only=(prediction_regions_org[:,:]==1)*1 - + mask_images_only=(prediction_regions_org[:,:]==2)*1 - - - + + + pixel_img=1 min_area_text=0.00001 - polygons_of_only_texts=self.return_contours_of_interested_region(mask_texts_only,pixel_img,min_area_text) - - polygons_of_only_images=self.return_contours_of_interested_region(mask_images_only,pixel_img) - - polygons_of_only_lines=self.return_contours_of_interested_region(mask_lines_only,pixel_img,min_area_text) - - - text_regions_p_true=np.zeros(prediction_regions_org.shape) - #text_regions_p_true[:,:]=text_regions_p_1[:,:] - - text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_lines, color=(3,3,3)) - - ##text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_images, color=(2,2,2)) - text_regions_p_true[:,:][mask_images_only[:,:]==1]=2 - - text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) - - ##print(np.unique(text_regions_p_true)) - - - #text_regions_p_true_3d=np.repeat(text_regions_p_1[:, :, np.newaxis], 3, axis=2) - #text_regions_p_true_3d=text_regions_p_true_3d.astype(np.uint8) - - del polygons_of_only_texts - del polygons_of_only_images - del polygons_of_only_lines - del mask_images_only - del prediction_regions_org - del img - del mask_zeros_y - - del prediction_regions_org_y - del img_org - gc.collect() - - return text_regions_p_true - - - def get_textregion_contours_in_org_image(self,cnts,img,slope_first): - - cnts_org=[] - #print(cnts,'cnts') - for i in range(len(cnts)): - img_copy=np.zeros(img.shape) - img_copy=cv2.fillPoly(img_copy,pts=[cnts[i]], color=(1,1,1)) - - #plt.imshow(img_copy) - #plt.show() - - #print(img.shape,'img') - img_copy=self.rotation_image_new(img_copy,-slope_first) - ##print(img_copy.shape,'img_copy') - #plt.imshow(img_copy) - #plt.show() - - img_copy=img_copy.astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - cont_int,_=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - cont_int[0][:,0,0]=cont_int[0][:,0,0]+np.abs(img_copy.shape[1]-img.shape[1]) - cont_int[0][:,0,1]=cont_int[0][:,0,1]+np.abs(img_copy.shape[0]-img.shape[0]) - #print(np.shape(cont_int[0])) - cnts_org.append(cont_int[0]) - - - - #print(cnts_org,'cnts_org') - - #sys.exit() - self.y_shift=np.abs(img_copy.shape[0]-img.shape[0]) - self.x_shift=np.abs(img_copy.shape[1]-img.shape[1]) - return cnts_org - def write_images_into_directory(self,img_contoures,dir_of_cropped_imgs,image_page): - index=0 - for cont_ind in img_contoures: - #cont_ind[:,0,0]=cont_ind[:,0,0]/self.scale_x - #cont_ind[:,0,1]=cont_ind[:,0,1]/self.scale_y - - x, y, w, h = cv2.boundingRect(cont_ind) - box = [x, y, w, h] - croped_page, page_coord = self.crop_image_inside_box(box, image_page) - - croped_page = self.resize_image(croped_page, int(croped_page.shape[0]/self.scale_y), int(croped_page.shape[1]/self.scale_x)) - - path=os.path.join(dir_of_cropped_imgs, self.f_name+'_'+str(index)+'.jpg') - cv2.imwrite(path,croped_page) - index+=1 - - def get_marginals(self,text_with_lines,text_regions,num_col,slope_deskew): - mask_marginals=np.zeros((text_with_lines.shape[0],text_with_lines.shape[1])) - mask_marginals=mask_marginals.astype(np.uint8) - - - text_with_lines=text_with_lines.astype(np.uint8) - ##text_with_lines=cv2.erode(text_with_lines,self.kernel,iterations=3) - - text_with_lines_eroded=cv2.erode(text_with_lines,self.kernel,iterations=5) - - if text_with_lines.shape[0]<=1500: - pass - elif text_with_lines.shape[0]>1500 and text_with_lines.shape[0]<=1800: - text_with_lines=self.resize_image(text_with_lines,int(text_with_lines.shape[0]*1.5),text_with_lines.shape[1]) - text_with_lines=cv2.erode(text_with_lines,self.kernel,iterations=5) - text_with_lines=self.resize_image(text_with_lines,text_with_lines_eroded.shape[0],text_with_lines_eroded.shape[1]) - else: - text_with_lines=self.resize_image(text_with_lines,int(text_with_lines.shape[0]*1.8),text_with_lines.shape[1]) - text_with_lines=cv2.erode(text_with_lines,self.kernel,iterations=7) - text_with_lines=self.resize_image(text_with_lines,text_with_lines_eroded.shape[0],text_with_lines_eroded.shape[1]) - + polygons_of_only_texts=return_contours_of_interested_region(mask_texts_only,pixel_img,min_area_text) - text_with_lines_y=text_with_lines.sum(axis=0) - text_with_lines_y_eroded=text_with_lines_eroded.sum(axis=0) - - thickness_along_y_percent=text_with_lines_y_eroded.max()/(float(text_with_lines.shape[0]))*100 - - #print(thickness_along_y_percent,'thickness_along_y_percent') - - if thickness_along_y_percent<30: - min_textline_thickness=8 - elif thickness_along_y_percent>=30 and thickness_along_y_percent<50: - min_textline_thickness=20 - else: - min_textline_thickness=40 - - - - if thickness_along_y_percent>=14: - - text_with_lines_y_rev=-1*text_with_lines_y[:] - #print(text_with_lines_y) - #print(text_with_lines_y_rev) - + polygons_of_only_images=return_contours_of_interested_region(mask_images_only,pixel_img) - - - #plt.plot(text_with_lines_y) - #plt.show() - - - text_with_lines_y_rev=text_with_lines_y_rev-np.min(text_with_lines_y_rev) - - #plt.plot(text_with_lines_y_rev) - #plt.show() - sigma_gaus=1 - region_sum_0= gaussian_filter1d(text_with_lines_y, sigma_gaus) - - region_sum_0_rev=gaussian_filter1d(text_with_lines_y_rev, sigma_gaus) - - #plt.plot(region_sum_0_rev) - #plt.show() - region_sum_0_updown=region_sum_0[len(region_sum_0)::-1] + polygons_of_only_lines=return_contours_of_interested_region(mask_lines_only,pixel_img,min_area_text) - first_nonzero=(next((i for i, x in enumerate(region_sum_0) if x), None)) - last_nonzero=(next((i for i, x in enumerate(region_sum_0_updown) if x), None)) + text_regions_p_true=np.zeros(prediction_regions_org.shape) + #text_regions_p_true[:,:]=text_regions_p_1[:,:] - last_nonzero=len(region_sum_0)-last_nonzero - - ##img_sum_0_smooth_rev=-region_sum_0 - - - mid_point=(last_nonzero+first_nonzero)/2. - - - one_third_right=(last_nonzero-mid_point)/3.0 - one_third_left=(mid_point-first_nonzero)/3.0 - - #img_sum_0_smooth_rev=img_sum_0_smooth_rev-np.min(img_sum_0_smooth_rev) - + text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_lines, color=(3,3,3)) - - - peaks, _ = find_peaks(text_with_lines_y_rev, height=0) - + ##text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_images, color=(2,2,2)) + text_regions_p_true[:,:][mask_images_only[:,:]==1]=2 - peaks=np.array(peaks) - - - #print(region_sum_0[peaks]) - ##plt.plot(region_sum_0) - ##plt.plot(peaks,region_sum_0[peaks],'*') - ##plt.show() - #print(first_nonzero,last_nonzero,peaks) - peaks=peaks[(peaks>first_nonzero) & ((peaksmid_point] - peaks_left=peaks[peaks(mid_point+one_third_right)] - peaks_left=peaks[peaks<(mid_point-one_third_left)] - - - try: - point_right=np.min(peaks_right) - except: - point_right=last_nonzero - - - try: - point_left=np.max(peaks_left) - except: - point_left=first_nonzero - - + text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) - - #print(point_left,point_right) - #print(text_regions.shape) - if point_right>=mask_marginals.shape[1]: - point_right=mask_marginals.shape[1]-1 - - try: - mask_marginals[:,point_left:point_right]=1 - except: - mask_marginals[:,:]=1 - - #print(mask_marginals.shape,point_left,point_right,'nadosh') - mask_marginals_rotated=self.rotate_image(mask_marginals,-slope_deskew) - - #print(mask_marginals_rotated.shape,'nadosh') - mask_marginals_rotated_sum=mask_marginals_rotated.sum(axis=0) - - mask_marginals_rotated_sum[mask_marginals_rotated_sum!=0]=1 - index_x=np.array(range(len(mask_marginals_rotated_sum)))+1 - - index_x_interest=index_x[mask_marginals_rotated_sum==1] - - min_point_of_left_marginal=np.min(index_x_interest)-16 - max_point_of_right_marginal=np.max(index_x_interest)+16 - - if min_point_of_left_marginal<0: - min_point_of_left_marginal=0 - if max_point_of_right_marginal>=text_regions.shape[1]: - max_point_of_right_marginal=text_regions.shape[1]-1 - - - #print(np.min(index_x_interest) ,np.max(index_x_interest),'minmaxnew') - #print(mask_marginals_rotated.shape,text_regions.shape,'mask_marginals_rotated') - #plt.imshow(mask_marginals) - #plt.show() - - #plt.imshow(mask_marginals_rotated) - #plt.show() + ##print(np.unique(text_regions_p_true)) - text_regions[(mask_marginals_rotated[:,:]!=1) & (text_regions[:,:]==1)]=4 - - #plt.imshow(text_regions) - #plt.show() - - pixel_img=4 - min_area_text=0.00001 - polygons_of_marginals=self.return_contours_of_interested_region(text_regions,pixel_img,min_area_text) - - cx_text_only,cy_text_only ,x_min_text_only,x_max_text_only, y_min_text_only ,y_max_text_only,y_cor_x_min_main=self.find_new_features_of_contoures(polygons_of_marginals) - - text_regions[(text_regions[:,:]==4)]=1 - - marginlas_should_be_main_text=[] - - x_min_marginals_left=[] - x_min_marginals_right=[] - - for i in range(len(cx_text_only)): - - x_width_mar=abs(x_min_text_only[i]-x_max_text_only[i]) - y_height_mar=abs(y_min_text_only[i]-y_max_text_only[i]) - #print(x_width_mar,y_height_mar,y_height_mar/x_width_mar,'y_height_mar') - if x_width_mar>16 and y_height_mar/x_width_mar<18: - marginlas_should_be_main_text.append(polygons_of_marginals[i]) - if x_min_text_only[i]<(mid_point-one_third_left): - x_min_marginals_left_new=x_min_text_only[i] - if len(x_min_marginals_left)==0: - x_min_marginals_left.append(x_min_marginals_left_new) - else: - x_min_marginals_left[0]=min(x_min_marginals_left[0],x_min_marginals_left_new) - else: - x_min_marginals_right_new=x_min_text_only[i] - if len(x_min_marginals_right)==0: - x_min_marginals_right.append(x_min_marginals_right_new) - else: - x_min_marginals_right[0]=min(x_min_marginals_right[0],x_min_marginals_right_new) - - if len(x_min_marginals_left)==0: - x_min_marginals_left=[0] - if len(x_min_marginals_right)==0: - x_min_marginals_right=[text_regions.shape[1]-1] - - - - - #print(x_min_marginals_left[0],x_min_marginals_right[0],'margo') - - #print(marginlas_should_be_main_text,'marginlas_should_be_main_text') - text_regions=cv2.fillPoly(text_regions, pts =marginlas_should_be_main_text, color=(4,4)) - - #print(np.unique(text_regions)) - - #text_regions[:,:int(x_min_marginals_left[0])][text_regions[:,:int(x_min_marginals_left[0])]==1]=0 - #text_regions[:,int(x_min_marginals_right[0]):][text_regions[:,int(x_min_marginals_right[0]):]==1]=0 - - text_regions[:,:int(min_point_of_left_marginal)][text_regions[:,:int(min_point_of_left_marginal)]==1]=0 - text_regions[:,int(max_point_of_right_marginal):][text_regions[:,int(max_point_of_right_marginal):]==1]=0 - - ###text_regions[:,0:point_left][text_regions[:,0:point_left]==1]=4 - - ###text_regions[:,point_right:][ text_regions[:,point_right:]==1]=4 - #plt.plot(region_sum_0) - #plt.plot(peaks,region_sum_0[peaks],'*') - #plt.show() - - - #plt.imshow(text_regions) - #plt.show() - - #sys.exit() - else: - pass - return text_regions - - def do_work_of_textline_seperation(self,queue_of_all_params,polygons_per_process, index_polygons_per_process,con_par_org,textline_mask_tot,mask_texts_only,num_col,scale_par,boxes_text): - - - textregions_cnt_tot_per_process=[] - textlines_cnt_tot_per_process=[] - index_polygons_per_process_per_process=[] - polygons_per_par_process_per_process=[] - textline_cnt_seperated=np.zeros(textline_mask_tot.shape) - for iiii in range(len(polygons_per_process)): - #crop_img,crop_coor=self.crop_image_inside_box(boxes_text[mv],image_page_rotated) - #arg_max=np.argmax(areas_cnt_only_text) - textregions_cnt_tot_per_process.append(polygons_per_process[iiii]/scale_par) - textline_region_in_image=np.zeros(textline_mask_tot.shape) - cnt_o_t_max=polygons_per_process[iiii] - - x, y, w, h = cv2.boundingRect(cnt_o_t_max) - - - mask_biggest=np.zeros(mask_texts_only.shape) - mask_biggest=cv2.fillPoly(mask_biggest, pts =[cnt_o_t_max], color=(1,1,1)) - - mask_region_in_patch_region=mask_biggest[y:y+h,x:x+w] - - - textline_biggest_region=mask_biggest*textline_mask_tot - - textline_rotated_seperated=self.seperate_lines_new2(textline_biggest_region[y:y+h,x:x+w],0,num_col) - - - #new line added - ##print(np.shape(textline_rotated_seperated),np.shape(mask_biggest)) - textline_rotated_seperated[mask_region_in_patch_region[:,:]!=1]=0 - #till here - - textline_cnt_seperated[y:y+h,x:x+w]=textline_rotated_seperated - textline_region_in_image[y:y+h,x:x+w]=textline_rotated_seperated - - #plt.imshow(textline_region_in_image) - #plt.show() - - #plt.imshow(textline_cnt_seperated) - #plt.show() - + #text_regions_p_true_3d=np.repeat(text_regions_p_1[:, :, np.newaxis], 3, axis=2) + #text_regions_p_true_3d=text_regions_p_true_3d.astype(np.uint8) - - - pixel_img=1 - cnt_textlines_in_image=self.return_contours_of_interested_textline(textline_region_in_image,pixel_img) - - - - textlines_cnt_per_region=[] - for jjjj in range(len(cnt_textlines_in_image)): - mask_biggest2=np.zeros(mask_texts_only.shape) - mask_biggest2=cv2.fillPoly(mask_biggest2, pts =[cnt_textlines_in_image[jjjj]], color=(1,1,1)) - if num_col+1==1: - mask_biggest2=cv2.dilate(mask_biggest2, self.kernel, iterations=5) - else: - - mask_biggest2=cv2.dilate(mask_biggest2, self.kernel, iterations=4) - - pixel_img=1 - cnt_textlines_in_image_ind=self.return_contours_of_interested_textline(mask_biggest2,pixel_img) - - try: - textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]/scale_par) - except: - pass - #print(len(cnt_textlines_in_image_ind)) - - - - #plt.imshow(mask_biggest2) - #plt.show() - textlines_cnt_tot_per_process.append(textlines_cnt_per_region) - index_polygons_per_process_per_process.append(index_polygons_per_process[iiii]) - polygons_per_par_process_per_process.append(con_par_org[iiii]) - + del polygons_of_only_texts + del polygons_of_only_images + del polygons_of_only_lines + del mask_images_only + del prediction_regions_org + del img + del mask_zeros_y - - queue_of_all_params.put([index_polygons_per_process_per_process,polygons_per_par_process_per_process,textregions_cnt_tot_per_process, textlines_cnt_tot_per_process ]) - - def small_textlines_to_parent_adherence2(self,textlines_con,textline_iamge,num_col): - #print(textlines_con) - #textlines_con=textlines_con.astype(np.uint32) - - textlines_con_changed=[] - for m1 in range(len(textlines_con)): - - #textlines_tot=textlines_con[m1] - #textlines_tot=textlines_tot.astype() - textlines_tot=[] - textlines_tot_org_form=[] - #print(textlines_tot) - - for nn in range(len(textlines_con[m1])): - textlines_tot.append(np.array( textlines_con[m1][nn],dtype=np.int32) ) - textlines_tot_org_form.append(textlines_con[m1][nn]) - - ##img_text_all=np.zeros((textline_iamge.shape[0],textline_iamge.shape[1])) - ##img_text_all=cv2.fillPoly(img_text_all, pts =textlines_tot , color=(1,1,1)) - - ##plt.imshow(img_text_all) - ##plt.show() - areas_cnt_text=np.array([cv2.contourArea(textlines_tot[j]) for j in range(len(textlines_tot))]) - areas_cnt_text=areas_cnt_text/float(textline_iamge.shape[0]*textline_iamge.shape[1]) - indexes_textlines=np.array(range(len(textlines_tot))) - - #print(areas_cnt_text,np.min(areas_cnt_text),np.max(areas_cnt_text)) - if num_col==0: - min_area=0.0004 - elif num_col==1: - min_area=0.0003 - else: - min_area=0.0001 - indexes_textlines_small=indexes_textlines[areas_cnt_text0: - - dis_small_from_bigs_tot=[] - for z1 in range(len(textlines_small)): - #print(len(textlines_small),'small') - intersections=[] - for z2 in range(len(textlines_big)): - img_text=np.zeros((textline_iamge.shape[0],textline_iamge.shape[1])) - img_text=cv2.fillPoly(img_text, pts =[textlines_small[z1] ] , color=(1,1,1)) - - img_text2=np.zeros((textline_iamge.shape[0],textline_iamge.shape[1])) - img_text2=cv2.fillPoly(img_text2, pts =[textlines_big[z2] ] , color=(1,1,1)) - - sum_small_big=img_text2+img_text - sum_small_big_2=(sum_small_big[:,:]==2)*1 - - sum_intersection=sum_small_big_2.sum(axis=1).sum() - - #print(sum_intersection) - - intersections.append(sum_intersection) - - if len(np.array(intersections)[np.array(intersections)>0] )==0: - intersections=[] - - try: - dis_small_from_bigs_tot.append(np.argmax(intersections)) - except: - dis_small_from_bigs_tot.append(-1) - - - smalls_list=np.array(dis_small_from_bigs_tot)[np.array(dis_small_from_bigs_tot)>=0] - - #index_small_textlines_rest=list( set(indexes_textlines_small)-set(smalls_list) ) - - textlines_big_with_change=[] - textlines_big_with_change_con=[] - textlines_small_with_change=[] - - - for z in list(set(smalls_list)): - index_small_textlines=list(np.where(np.array(dis_small_from_bigs_tot)==z)[0]) - #print(z,index_small_textlines) - - img_text2=np.zeros((textline_iamge.shape[0],textline_iamge.shape[1],3)) - img_text2=cv2.fillPoly(img_text2, pts =[textlines_big[z]] , color=(255,255,255)) - - textlines_big_with_change.append(z) - - for k in index_small_textlines: - img_text2=cv2.fillPoly(img_text2, pts =[textlines_small[k]] , color=(255,255,255)) - textlines_small_with_change.append(k) - - - img_text2=img_text2.astype(np.uint8) - imgray = cv2.cvtColor(img_text2, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - cont,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - #print(cont[0],type(cont)) - - textlines_big_with_change_con.append(cont) - textlines_big_org_form[z]=cont[0] - - #plt.imshow(img_text2) - #plt.show() - - #print(textlines_big_with_change,'textlines_big_with_change') - #print(textlines_small_with_change,'textlines_small_with_change') - #print(textlines_big) - textlines_con_changed.append(textlines_big_org_form) - - else: - textlines_con_changed.append(textlines_big_org_form) - return textlines_con_changed - def check_any_text_region_in_model_one_is_main_or_header(self,regions_model_1,regions_model_full,contours_only_text_parent,all_box_coord,all_found_texline_polygons,slopes,contours_only_text_parent_d_ordered): - #text_only=(regions_model_1[:,:]==1)*1 - #contours_only_text,hir_on_text=self.return_contours_of_image(text_only) - - """ - contours_only_text_parent=self.return_parent_contours( contours_only_text,hir_on_text) - - areas_cnt_text=np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) - areas_cnt_text=areas_cnt_text/float(text_only.shape[0]*text_only.shape[1]) - - ###areas_cnt_text_h=np.array([cv2.contourArea(contours_only_text_parent_h[j]) for j in range(len(contours_only_text_parent_h))]) - ###areas_cnt_text_h=areas_cnt_text_h/float(text_only_h.shape[0]*text_only_h.shape[1]) + del prediction_regions_org_y + del img_org + gc.collect() - ###contours_only_text_parent=[contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz]>0.0002] - contours_only_text_parent=[contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz]>0.00001] - """ - - cx_main,cy_main ,x_min_main , x_max_main, y_min_main ,y_max_main,y_corr_x_min_from_argmin=self.find_new_features_of_contoures(contours_only_text_parent) - - length_con=x_max_main-x_min_main - height_con=y_max_main-y_min_main - - + return text_regions_p_true - all_found_texline_polygons_main=[] - all_found_texline_polygons_head=[] - - all_box_coord_main=[] - all_box_coord_head=[] - - slopes_main=[] - slopes_head=[] - - contours_only_text_parent_main=[] - contours_only_text_parent_head=[] - - contours_only_text_parent_main_d=[] - contours_only_text_parent_head_d=[] - - for ii in range(len(contours_only_text_parent)): - con=contours_only_text_parent[ii] - img=np.zeros((regions_model_1.shape[0],regions_model_1.shape[1],3)) - img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) - - - - all_pixels=((img[:,:,0]==255)*1).sum() - - pixels_header=( ( (img[:,:,0]==255) & (regions_model_full[:,:,0]==2) )*1 ).sum() - pixels_main=all_pixels-pixels_header - - - if (pixels_header>=pixels_main) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 - contours_only_text_parent_head.append(con) - if contours_only_text_parent_d_ordered is not None: - contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) - all_box_coord_head.append(all_box_coord[ii]) - slopes_head.append(slopes[ii]) - all_found_texline_polygons_head.append(all_found_texline_polygons[ii]) - else: - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 - contours_only_text_parent_main.append(con) - if contours_only_text_parent_d_ordered is not None: - contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) - all_box_coord_main.append(all_box_coord[ii]) - slopes_main.append(slopes[ii]) - all_found_texline_polygons_main.append(all_found_texline_polygons[ii]) - - #print(all_pixels,pixels_main,pixels_header) - - - #plt.imshow(img[:,:,0]) - #plt.show() - return regions_model_1,contours_only_text_parent_main,contours_only_text_parent_head,all_box_coord_main,all_box_coord_head,all_found_texline_polygons_main,all_found_texline_polygons_head,slopes_main,slopes_head,contours_only_text_parent_main_d,contours_only_text_parent_head_d - - def putt_bb_of_drop_capitals_of_model_in_patches_in_layout(self,layout_in_patch): - - drop_only=(layout_in_patch[:,:,0]==4)*1 - contours_drop,hir_on_drop=self.return_contours_of_image(drop_only) - contours_drop_parent=self.return_parent_contours( contours_drop,hir_on_drop) - - areas_cnt_text=np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) - areas_cnt_text=areas_cnt_text/float(drop_only.shape[0]*drop_only.shape[1]) - - contours_drop_parent=[contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz]>0.00001] - - areas_cnt_text=[areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz]>0.001] - - - contours_drop_parent_final=[] - - for jj in range(len(contours_drop_parent)): - x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) - layout_in_patch[y:y+h,x:x+w,0]=4 - - return layout_in_patch - - def filter_small_drop_capitals_from_no_patch_layout(self,layout_no_patch,layout1): - - drop_only=(layout_no_patch[:,:,0]==4)*1 - contours_drop,hir_on_drop=self.return_contours_of_image(drop_only) - contours_drop_parent=self.return_parent_contours( contours_drop,hir_on_drop) - - areas_cnt_text=np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) - areas_cnt_text=areas_cnt_text/float(drop_only.shape[0]*drop_only.shape[1]) - - contours_drop_parent=[contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz]>0.001] - - areas_cnt_text=[areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz]>0.001] - - - contours_drop_parent_final=[] - - for jj in range(len(contours_drop_parent)): - x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) - #boxes.append([int(x), int(y), int(w), int(h)]) - - iou_of_box_and_contoure=float(drop_only.shape[0]*drop_only.shape[1])*areas_cnt_text[jj]/float(w*h)*100 - height_to_weight_ratio=h/float(w) - weigh_to_height_ratio=w/float(h) - - if iou_of_box_and_contoure>60 and weigh_to_height_ratio<1.2 and height_to_weight_ratio<2: - map_of_drop_contour_bb=np.zeros((layout1.shape[0],layout1.shape[1])) - map_of_drop_contour_bb[y:y+h,x:x+w]=layout1[y:y+h,x:x+w] - - if ( ((map_of_drop_contour_bb==1)*1).sum()/float(((map_of_drop_contour_bb==5)*1).sum()) *100)>=15: - contours_drop_parent_final.append(contours_drop_parent[jj]) - - layout_no_patch[:,:,0][layout_no_patch[:,:,0]==4]=0 - - layout_no_patch=cv2.fillPoly(layout_no_patch, pts=contours_drop_parent_final, color=(4,4, 4)) - - return layout_no_patch - - - def put_drop_out_from_only_drop_model(self,layout_no_patch,layout1): - - drop_only=(layout_no_patch[:,:,0]==4)*1 - contours_drop,hir_on_drop=self.return_contours_of_image(drop_only) - contours_drop_parent=self.return_parent_contours( contours_drop,hir_on_drop) - - areas_cnt_text=np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) - areas_cnt_text=areas_cnt_text/float(drop_only.shape[0]*drop_only.shape[1]) - - contours_drop_parent=[contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz]>0.00001] - - areas_cnt_text=[areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz]>0.00001] - - - contours_drop_parent_final=[] - - for jj in range(len(contours_drop_parent)): - x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) - #boxes.append([int(x), int(y), int(w), int(h)]) - - map_of_drop_contour_bb=np.zeros((layout1.shape[0],layout1.shape[1])) - map_of_drop_contour_bb[y:y+h,x:x+w]=layout1[y:y+h,x:x+w] - - if ( ((map_of_drop_contour_bb==1)*1).sum()/float(((map_of_drop_contour_bb==5)*1).sum()) *100)>=15: - contours_drop_parent_final.append(contours_drop_parent[jj]) - - layout_no_patch[:,:,0][layout_no_patch[:,:,0]==4]=0 - - layout_no_patch=cv2.fillPoly(layout_no_patch, pts=contours_drop_parent_final, color=(4,4, 4)) - - return layout_no_patch - - def do_order_of_regions(self,contours_only_text_parent,contours_only_text_parent_h,boxes,textline_mask_tot): - - if self.full_layout=='true' or self.full_layout=='True' or self.full_layout=='TRUE': - cx_text_only,cy_text_only ,x_min_text_only, _, _ ,_,y_cor_x_min_main=self.find_new_features_of_contoures(contours_only_text_parent) - cx_text_only_h,cy_text_only_h ,x_min_text_only_h, _, _ ,_,y_cor_x_min_main_h=self.find_new_features_of_contoures(contours_only_text_parent_h) + def write_images_into_directory(self, img_contoures, dir_of_cropped_imgs, image_page): + index = 0 + for cont_ind in img_contoures: + # cont_ind[:,0,0]=cont_ind[:,0,0]/self.scale_x + # cont_ind[:,0,1]=cont_ind[:,0,1]/self.scale_y + + x, y, w, h = cv2.boundingRect(cont_ind) + box = [x, y, w, h] + croped_page, page_coord = crop_image_inside_box(box, image_page) + + croped_page = resize_image(croped_page, int(croped_page.shape[0] / self.scale_y), int(croped_page.shape[1] / self.scale_x)) + + path = os.path.join(dir_of_cropped_imgs, self.f_name + "_" + str(index) + ".jpg") + cv2.imwrite(path, croped_page) + index += 1 + + def do_order_of_regions(self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): + + if self.full_layout: + cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contoures(contours_only_text_parent) + cx_text_only_h, cy_text_only_h, x_min_text_only_h, _, _, _, y_cor_x_min_main_h = find_new_features_of_contoures(contours_only_text_parent_h) - try: - arg_text_con=[] + arg_text_con = [] for ii in range(len(cx_text_only)): for jj in range(len(boxes)): - if (x_min_text_only[ii]+80) >=boxes[jj][0] and (x_min_text_only[ii]+80) < boxes[jj][1] and y_cor_x_min_main[ii] >=boxes[jj][2] and y_cor_x_min_main[ii] < boxes[jj][3]: + if (x_min_text_only[ii] + 80) >= boxes[jj][0] and (x_min_text_only[ii] + 80) < boxes[jj][1] and y_cor_x_min_main[ii] >= boxes[jj][2] and y_cor_x_min_main[ii] < boxes[jj][3]: arg_text_con.append(jj) break - arg_arg_text_con=np.argsort(arg_text_con) - args_contours=np.array(range( len(arg_text_con)) ) - - arg_text_con_h=[] + arg_arg_text_con = np.argsort(arg_text_con) + args_contours = np.array(range(len(arg_text_con))) + + arg_text_con_h = [] for ii in range(len(cx_text_only_h)): for jj in range(len(boxes)): - if (x_min_text_only_h[ii]+80) >=boxes[jj][0] and (x_min_text_only_h[ii]+80) < boxes[jj][1] and y_cor_x_min_main_h[ii] >=boxes[jj][2] and y_cor_x_min_main_h[ii] < boxes[jj][3]: + if (x_min_text_only_h[ii] + 80) >= boxes[jj][0] and (x_min_text_only_h[ii] + 80) < boxes[jj][1] and y_cor_x_min_main_h[ii] >= boxes[jj][2] and y_cor_x_min_main_h[ii] < boxes[jj][3]: arg_text_con_h.append(jj) break - arg_arg_text_con=np.argsort(arg_text_con_h) - args_contours_h=np.array(range( len(arg_text_con_h)) ) - - - order_by_con_head=np.zeros(len(arg_text_con_h)) - order_by_con_main=np.zeros(len(arg_text_con)) + arg_arg_text_con = np.argsort(arg_text_con_h) + args_contours_h = np.array(range(len(arg_text_con_h))) + order_by_con_head = np.zeros(len(arg_text_con_h)) + order_by_con_main = np.zeros(len(arg_text_con)) - ref_point=0 - order_of_texts_tot=[] - id_of_texts_tot=[] + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] for iij in range(len(boxes)): - args_contours_box=args_contours[np.array(arg_text_con)==iij] - args_contours_box_h=args_contours_h[np.array(arg_text_con_h)==iij] - con_inter_box=[] - con_inter_box_h=[] + args_contours_box = args_contours[np.array(arg_text_con) == iij] + args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] + con_inter_box = [] + con_inter_box_h = [] for i in range(len(args_contours_box)): - con_inter_box.append( contours_only_text_parent[args_contours_box[i] ] ) - + con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) + for i in range(len(args_contours_box_h)): - con_inter_box_h.append( contours_only_text_parent_h[args_contours_box_h[i] ] ) - - indexes_sorted, matrix_of_orders,kind_of_texts_sorted,index_by_kind_sorted=self.order_of_regions(textline_mask_tot[int(boxes[iij][2]):int(boxes[iij][3]), int(boxes[iij][0]):int(boxes[iij][1])],con_inter_box,con_inter_box_h,boxes[iij][2]) - - - order_of_texts, id_of_texts=self.order_and_id_of_texts(con_inter_box ,con_inter_box_h,matrix_of_orders ,indexes_sorted ,index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - - indexes_sorted_main=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_by_type_main=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_sorted_head=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==2] - indexes_by_type_head=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==2] - - - - zahler=0 + con_inter_box_h.append(contours_only_text_parent_h[args_contours_box_h[i]]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + + zahler = 0 for mtv in args_contours_box: - arg_order_v=indexes_sorted_main[zahler] - tartib=np.where(indexes_sorted==arg_order_v )[0][0] - order_by_con_main[ args_contours_box[indexes_by_type_main[zahler] ]]=tartib+ref_point - zahler=zahler+1 - - zahler=0 + arg_order_v = indexes_sorted_main[zahler] + tartib = np.where(indexes_sorted == arg_order_v)[0][0] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = tartib + ref_point + zahler = zahler + 1 + + zahler = 0 for mtv in args_contours_box_h: - arg_order_v=indexes_sorted_head[zahler] - tartib=np.where(indexes_sorted==arg_order_v )[0][0] - #print(indexes_sorted,np.where(indexes_sorted==arg_order_v ),arg_order_v,tartib,'inshgalla') - order_by_con_head[ args_contours_box_h[indexes_by_type_head[zahler] ]]=tartib+ref_point - zahler=zahler+1 - - - + arg_order_v = indexes_sorted_head[zahler] + tartib = np.where(indexes_sorted == arg_order_v)[0][0] + # print(indexes_sorted,np.where(indexes_sorted==arg_order_v ),arg_order_v,tartib,'inshgalla') + order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = tartib + ref_point + zahler = zahler + 1 + for jji in range(len(id_of_texts)): - order_of_texts_tot.append(order_of_texts[jji]+ref_point) + order_of_texts_tot.append(order_of_texts[jji] + ref_point) id_of_texts_tot.append(id_of_texts[jji]) - ref_point=ref_point+len(id_of_texts) + ref_point = ref_point + len(id_of_texts) - - order_of_texts_tot=[] + order_of_texts_tot = [] for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1] )) + order_of_texts_tot.append(int(order_by_con_main[tj1])) for tj1 in range(len(contours_only_text_parent_h)): - order_of_texts_tot.append(int(order_by_con_head[tj1]) ) - - order_text_new=[] + order_of_texts_tot.append(int(order_by_con_head[tj1])) + + order_text_new = [] for iii in range(len(order_of_texts_tot)): - tartib_new=np.where(np.array(order_of_texts_tot)==iii)[0][0] + tartib_new = np.where(np.array(order_of_texts_tot) == iii)[0][0] order_text_new.append(tartib_new) - except: - arg_text_con=[] + arg_text_con = [] for ii in range(len(cx_text_only)): for jj in range(len(boxes)): - if cx_text_only[ii] >=boxes[jj][0] and cx_text_only[ii] < boxes[jj][1] and cy_text_only[ii] >=boxes[jj][2] and cy_text_only[ii] < boxes[jj][3]:#this is valid if the center of region identify in which box it is located + if cx_text_only[ii] >= boxes[jj][0] and cx_text_only[ii] < boxes[jj][1] and cy_text_only[ii] >= boxes[jj][2] and cy_text_only[ii] < boxes[jj][3]: # this is valid if the center of region identify in which box it is located arg_text_con.append(jj) break - arg_arg_text_con=np.argsort(arg_text_con) - args_contours=np.array(range( len(arg_text_con)) ) + arg_arg_text_con = np.argsort(arg_text_con) + args_contours = np.array(range(len(arg_text_con))) - - order_by_con_main=np.zeros(len(arg_text_con)) + order_by_con_main = np.zeros(len(arg_text_con)) - - ############################# head - - arg_text_con_h=[] + + arg_text_con_h = [] for ii in range(len(cx_text_only_h)): for jj in range(len(boxes)): - if cx_text_only_h[ii] >=boxes[jj][0] and cx_text_only_h[ii] < boxes[jj][1] and cy_text_only_h[ii] >=boxes[jj][2] and cy_text_only_h[ii] < boxes[jj][3]:#this is valid if the center of region identify in which box it is located + if cx_text_only_h[ii] >= boxes[jj][0] and cx_text_only_h[ii] < boxes[jj][1] and cy_text_only_h[ii] >= boxes[jj][2] and cy_text_only_h[ii] < boxes[jj][3]: # this is valid if the center of region identify in which box it is located arg_text_con_h.append(jj) break - arg_arg_text_con_h=np.argsort(arg_text_con_h) - args_contours_h=np.array(range( len(arg_text_con_h)) ) + arg_arg_text_con_h = np.argsort(arg_text_con_h) + args_contours_h = np.array(range(len(arg_text_con_h))) - order_by_con_head=np.zeros(len(arg_text_con_h)) + order_by_con_head = np.zeros(len(arg_text_con_h)) ##### - ref_point=0 - order_of_texts_tot=[] - id_of_texts_tot=[] + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] for iij in range(len(boxes)): - args_contours_box=args_contours[np.array(arg_text_con)==iij] - args_contours_box_h=args_contours_h[np.array(arg_text_con_h)==iij] - con_inter_box=[] - con_inter_box_h=[] + args_contours_box = args_contours[np.array(arg_text_con) == iij] + args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] + con_inter_box = [] + con_inter_box_h = [] for i in range(len(args_contours_box)): - con_inter_box.append( contours_only_text_parent[args_contours_box[i] ] ) + con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) for i in range(len(args_contours_box_h)): - con_inter_box_h.append( contours_only_text_parent_h[args_contours_box_h[i] ] ) - - indexes_sorted, matrix_of_orders,kind_of_texts_sorted,index_by_kind_sorted=self.order_of_regions(textline_mask_tot[int(boxes[iij][2]):int(boxes[iij][3]), int(boxes[iij][0]):int(boxes[iij][1])],con_inter_box,con_inter_box_h,boxes[iij][2]) - - - order_of_texts, id_of_texts=self.order_and_id_of_texts(con_inter_box ,con_inter_box_h,matrix_of_orders ,indexes_sorted ,index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - - indexes_sorted_main=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_by_type_main=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_sorted_head=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==2] - indexes_by_type_head=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==2] - - - - zahler=0 + con_inter_box_h.append(contours_only_text_parent_h[args_contours_box_h[i]]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + + zahler = 0 for mtv in args_contours_box: - arg_order_v=indexes_sorted_main[zahler] - tartib=np.where(indexes_sorted==arg_order_v )[0][0] - order_by_con_main[ args_contours_box[indexes_by_type_main[zahler] ]]=tartib+ref_point - zahler=zahler+1 - - - zahler=0 + arg_order_v = indexes_sorted_main[zahler] + tartib = np.where(indexes_sorted == arg_order_v)[0][0] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = tartib + ref_point + zahler = zahler + 1 + + zahler = 0 for mtv in args_contours_box_h: - arg_order_v=indexes_sorted_head[zahler] - tartib=np.where(indexes_sorted==arg_order_v )[0][0] - #print(indexes_sorted,np.where(indexes_sorted==arg_order_v ),arg_order_v,tartib,'inshgalla') - order_by_con_head[ args_contours_box_h[indexes_by_type_head[zahler] ]]=tartib+ref_point - zahler=zahler+1 - - - + arg_order_v = indexes_sorted_head[zahler] + tartib = np.where(indexes_sorted == arg_order_v)[0][0] + # print(indexes_sorted,np.where(indexes_sorted==arg_order_v ),arg_order_v,tartib,'inshgalla') + order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = tartib + ref_point + zahler = zahler + 1 + for jji in range(len(id_of_texts)): - order_of_texts_tot.append(order_of_texts[jji]+ref_point) + order_of_texts_tot.append(order_of_texts[jji] + ref_point) id_of_texts_tot.append(id_of_texts[jji]) - ref_point=ref_point+len(id_of_texts) - + ref_point = ref_point + len(id_of_texts) - - - order_of_texts_tot=[] + order_of_texts_tot = [] for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1] )) + order_of_texts_tot.append(int(order_by_con_main[tj1])) for tj1 in range(len(contours_only_text_parent_h)): - order_of_texts_tot.append(int(order_by_con_head[tj1]) ) + order_of_texts_tot.append(int(order_by_con_head[tj1])) - - order_text_new=[] + order_text_new = [] for iii in range(len(order_of_texts_tot)): - tartib_new=np.where(np.array(order_of_texts_tot)==iii)[0][0] + tartib_new = np.where(np.array(order_of_texts_tot) == iii)[0][0] order_text_new.append(tartib_new) - return order_text_new,id_of_texts_tot - + return order_text_new, id_of_texts_tot + else: - cx_text_only,cy_text_only ,x_min_text_only, _, _ ,_,y_cor_x_min_main=self.find_new_features_of_contoures(contours_only_text_parent) + cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contoures(contours_only_text_parent) - try: - arg_text_con=[] + arg_text_con = [] for ii in range(len(cx_text_only)): for jj in range(len(boxes)): - if (x_min_text_only[ii]+80) >=boxes[jj][0] and (x_min_text_only[ii]+80) < boxes[jj][1] and y_cor_x_min_main[ii] >=boxes[jj][2] and y_cor_x_min_main[ii] < boxes[jj][3]: + if (x_min_text_only[ii] + 80) >= boxes[jj][0] and (x_min_text_only[ii] + 80) < boxes[jj][1] and y_cor_x_min_main[ii] >= boxes[jj][2] and y_cor_x_min_main[ii] < boxes[jj][3]: arg_text_con.append(jj) break - arg_arg_text_con=np.argsort(arg_text_con) - args_contours=np.array(range( len(arg_text_con)) ) - - - - order_by_con_main=np.zeros(len(arg_text_con)) + arg_arg_text_con = np.argsort(arg_text_con) + args_contours = np.array(range(len(arg_text_con))) + order_by_con_main = np.zeros(len(arg_text_con)) - ref_point=0 - order_of_texts_tot=[] - id_of_texts_tot=[] + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] for iij in range(len(boxes)): - args_contours_box=args_contours[np.array(arg_text_con)==iij] + args_contours_box = args_contours[np.array(arg_text_con) == iij] - con_inter_box=[] - con_inter_box_h=[] + con_inter_box = [] + con_inter_box_h = [] for i in range(len(args_contours_box)): - con_inter_box.append( contours_only_text_parent[args_contours_box[i] ] ) - - - indexes_sorted, matrix_of_orders,kind_of_texts_sorted,index_by_kind_sorted=self.order_of_regions(textline_mask_tot[int(boxes[iij][2]):int(boxes[iij][3]), int(boxes[iij][0]):int(boxes[iij][1])],con_inter_box,con_inter_box_h,boxes[iij][2]) - - - order_of_texts, id_of_texts=self.order_and_id_of_texts(con_inter_box ,con_inter_box_h,matrix_of_orders ,indexes_sorted ,index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - - indexes_sorted_main=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_by_type_main=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_sorted_head=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==2] - indexes_by_type_head=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==2] - - - - zahler=0 + con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + + zahler = 0 for mtv in args_contours_box: - arg_order_v=indexes_sorted_main[zahler] - tartib=np.where(indexes_sorted==arg_order_v )[0][0] - order_by_con_main[ args_contours_box[indexes_by_type_main[zahler] ]]=tartib+ref_point - zahler=zahler+1 - - - - + arg_order_v = indexes_sorted_main[zahler] + tartib = np.where(indexes_sorted == arg_order_v)[0][0] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = tartib + ref_point + zahler = zahler + 1 + for jji in range(len(id_of_texts)): - order_of_texts_tot.append(order_of_texts[jji]+ref_point) + order_of_texts_tot.append(order_of_texts[jji] + ref_point) id_of_texts_tot.append(id_of_texts[jji]) - ref_point=ref_point+len(id_of_texts) + ref_point = ref_point + len(id_of_texts) - - order_of_texts_tot=[] + order_of_texts_tot = [] for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1] )) + order_of_texts_tot.append(int(order_by_con_main[tj1])) - - order_text_new=[] + order_text_new = [] for iii in range(len(order_of_texts_tot)): - tartib_new=np.where(np.array(order_of_texts_tot)==iii)[0][0] + tartib_new = np.where(np.array(order_of_texts_tot) == iii)[0][0] order_text_new.append(tartib_new) - except: - arg_text_con=[] + arg_text_con = [] for ii in range(len(cx_text_only)): for jj in range(len(boxes)): - if cx_text_only[ii] >=boxes[jj][0] and cx_text_only[ii] < boxes[jj][1] and cy_text_only[ii] >=boxes[jj][2] and cy_text_only[ii] < boxes[jj][3]:#this is valid if the center of region identify in which box it is located + if cx_text_only[ii] >= boxes[jj][0] and cx_text_only[ii] < boxes[jj][1] and cy_text_only[ii] >= boxes[jj][2] and cy_text_only[ii] < boxes[jj][3]: # this is valid if the center of region identify in which box it is located arg_text_con.append(jj) break - arg_arg_text_con=np.argsort(arg_text_con) - args_contours=np.array(range( len(arg_text_con)) ) + arg_arg_text_con = np.argsort(arg_text_con) + args_contours = np.array(range(len(arg_text_con))) - - order_by_con_main=np.zeros(len(arg_text_con)) - - + order_by_con_main = np.zeros(len(arg_text_con)) - ref_point=0 - order_of_texts_tot=[] - id_of_texts_tot=[] + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] for iij in range(len(boxes)): - args_contours_box=args_contours[np.array(arg_text_con)==iij] - con_inter_box=[] - con_inter_box_h=[] + args_contours_box = args_contours[np.array(arg_text_con) == iij] + con_inter_box = [] + con_inter_box_h = [] for i in range(len(args_contours_box)): - con_inter_box.append( contours_only_text_parent[args_contours_box[i] ] ) + con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) - - indexes_sorted, matrix_of_orders,kind_of_texts_sorted,index_by_kind_sorted=self.order_of_regions(textline_mask_tot[int(boxes[iij][2]):int(boxes[iij][3]), int(boxes[iij][0]):int(boxes[iij][1])],con_inter_box,con_inter_box_h,boxes[iij][2]) - + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) - order_of_texts, id_of_texts=self.order_and_id_of_texts(con_inter_box ,con_inter_box_h,matrix_of_orders ,indexes_sorted ,index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - - indexes_sorted_main=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_by_type_main=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==1] - indexes_sorted_head=np.array(indexes_sorted)[np.array(kind_of_texts_sorted)==2] - indexes_by_type_head=np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted)==2] - + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - zahler=0 - for mtv in args_contours_box: - arg_order_v=indexes_sorted_main[zahler] - tartib=np.where(indexes_sorted==arg_order_v )[0][0] - order_by_con_main[ args_contours_box[indexes_by_type_main[zahler] ]]=tartib+ref_point - zahler=zahler+1 + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + zahler = 0 + for mtv in args_contours_box: + arg_order_v = indexes_sorted_main[zahler] + tartib = np.where(indexes_sorted == arg_order_v)[0][0] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = tartib + ref_point + zahler = zahler + 1 - - for jji in range(len(id_of_texts)): - order_of_texts_tot.append(order_of_texts[jji]+ref_point) + order_of_texts_tot.append(order_of_texts[jji] + ref_point) id_of_texts_tot.append(id_of_texts[jji]) - ref_point=ref_point+len(id_of_texts) - + ref_point = ref_point + len(id_of_texts) - - - order_of_texts_tot=[] + order_of_texts_tot = [] for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1] )) + order_of_texts_tot.append(int(order_by_con_main[tj1])) - - order_text_new=[] + order_text_new = [] for iii in range(len(order_of_texts_tot)): - tartib_new=np.where(np.array(order_of_texts_tot)==iii)[0][0] + tartib_new = np.where(np.array(order_of_texts_tot) == iii)[0][0] order_text_new.append(tartib_new) - return order_text_new,id_of_texts_tot - def adhere_drop_capital_region_into_cprresponding_textline(self,text_regions_p,polygons_of_drop_capitals,contours_only_text_parent,contours_only_text_parent_h,all_box_coord,all_box_coord_h,all_found_texline_polygons,all_found_texline_polygons_h): - #print(np.shape(all_found_texline_polygons),np.shape(all_found_texline_polygons[3]),'all_found_texline_polygonsshape') - #print(all_found_texline_polygons[3]) - cx_m,cy_m ,_, _, _ ,_,_=self.find_new_features_of_contoures(contours_only_text_parent) - cx_h,cy_h ,_, _, _ ,_,_=self.find_new_features_of_contoures(contours_only_text_parent_h) - cx_d,cy_d ,_ , _, y_min_d ,y_max_d,_=self.find_new_features_of_contoures(polygons_of_drop_capitals) - - img_con_all=np.zeros((text_regions_p.shape[0],text_regions_p.shape[1],3)) - for j_cont in range(len(contours_only_text_parent)): - img_con_all[all_box_coord[j_cont][0]:all_box_coord[j_cont][1],all_box_coord[j_cont][2]:all_box_coord[j_cont][3],0]=(j_cont+1)*3 - #img_con_all=cv2.fillPoly(img_con_all,pts=[contours_only_text_parent[j_cont]],color=((j_cont+1)*3,(j_cont+1)*3,(j_cont+1)*3)) - - #plt.imshow(img_con_all[:,:,0]) - #plt.show() - #img_con_all=cv2.dilate(img_con_all, self.kernel, iterations=3) - - #plt.imshow(img_con_all[:,:,0]) - #plt.show() - #print(np.unique(img_con_all[:,:,0])) - for i_drop in range(len(polygons_of_drop_capitals)): - #print(i_drop,'i_drop') - img_con_all_copy=np.copy(img_con_all) - img_con=np.zeros((text_regions_p.shape[0],text_regions_p.shape[1],3)) - img_con=cv2.fillPoly(img_con,pts=[polygons_of_drop_capitals[i_drop] ],color=(1,1,1)) - - - #plt.imshow(img_con[:,:,0]) - #plt.show() - ##img_con=cv2.dilate(img_con, self.kernel, iterations=30) - - #plt.imshow(img_con[:,:,0]) - #plt.show() - - #print(np.unique(img_con[:,:,0])) - - img_con_all_copy[:,:,0]=img_con_all_copy[:,:,0]+img_con[:,:,0] - - img_con_all_copy[:,:,0][img_con_all_copy[:,:,0]==1]=0 - - kherej_ghesmat=np.unique(img_con_all_copy[:,:,0])/3 - res_summed_pixels=np.unique(img_con_all_copy[:,:,0])%3 - region_with_intersected_drop=kherej_ghesmat[res_summed_pixels==1] - #region_with_intersected_drop=region_with_intersected_drop/3 - region_with_intersected_drop=region_with_intersected_drop.astype(np.uint8) - - #print(len(region_with_intersected_drop),'region_with_intersected_drop1') - if len(region_with_intersected_drop)==0: - img_con_all_copy=np.copy(img_con_all) - img_con=cv2.dilate(img_con, self.kernel, iterations=4) - - img_con_all_copy[:,:,0]=img_con_all_copy[:,:,0]+img_con[:,:,0] - - img_con_all_copy[:,:,0][img_con_all_copy[:,:,0]==1]=0 - - kherej_ghesmat=np.unique(img_con_all_copy[:,:,0])/3 - res_summed_pixels=np.unique(img_con_all_copy[:,:,0])%3 - region_with_intersected_drop=kherej_ghesmat[res_summed_pixels==1] - #region_with_intersected_drop=region_with_intersected_drop/3 - region_with_intersected_drop=region_with_intersected_drop.astype(np.uint8) - #print(np.unique(img_con_all_copy[:,:,0])) - if self.curved_line=='True' or self.curved_line=='true': - - - if len(region_with_intersected_drop)>1: - sum_pixels_of_intersection=[] - for i in range(len(region_with_intersected_drop)): - #print((region_with_intersected_drop[i]*3+1)) - sum_pixels_of_intersection.append( ((img_con_all_copy[:,:,0]==(region_with_intersected_drop[i]*3+1))*1).sum() ) - #print(sum_pixels_of_intersection) - region_final=region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)]-1 - - #print(region_final,'region_final') - #cx_t,cy_t ,_, _, _ ,_,_=self.find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) - try: - cx_t,cy_t ,_, _, _ ,_,_=self.find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) - #print(all_box_coord[j_cont]) - #print(cx_t) - #print(cy_t) - #print(cx_d[i_drop]) - #print(cy_d[i_drop]) - y_lines=np.array(cy_t)#all_box_coord[int(region_final)][0]+np.array(cy_t) - - - #print(y_lines) - - y_lines[y_lines1: - sum_pixels_of_intersection=[] - for i in range(len(region_with_intersected_drop)): - #print((region_with_intersected_drop[i]*3+1)) - sum_pixels_of_intersection.append( ((img_con_all_copy[:,:,0]==(region_with_intersected_drop[i]*3+1))*1).sum() ) - #print(sum_pixels_of_intersection) - region_final=region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)]-1 - - #print(region_final,'region_final') - #cx_t,cy_t ,_, _, _ ,_,_=self.find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) - try: - cx_t,cy_t ,_, _, _ ,_,_=self.find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) - #print(all_box_coord[j_cont]) - #print(cx_t) - #print(cy_t) - #print(cx_d[i_drop]) - #print(cy_d[i_drop]) - y_lines=all_box_coord[int(region_final)][0]+np.array(cy_t) - - - #print(y_lines) - - y_lines[y_lines=.13: - image_page_rotated_n,textline_mask_tot_d,text_regions_p_1_n=self.rotation_not_90_func(image_page,textline_mask_tot,text_regions_p,slope_deskew) - - text_regions_p_1_n=self.resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) - textline_mask_tot_d=self.resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1]) - - regions_without_seperators_d=(text_regions_p_1_n[:,:]==1)*1 - - - - - - - regions_without_seperators=(text_regions_p[:,:]==1)*1#( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_seperators_new(text_regions_p[:,:,0],img_only_regions) - - - - pixel_lines=3 - if np.abs(slope_deskew)<0.13: - num_col,peaks_neg_fin,matrix_of_lines_ch,spliter_y_new,seperators_closeup_n=self.find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines) - - if np.abs(slope_deskew)>=.13: - num_col_d,peaks_neg_fin_d,matrix_of_lines_ch_d,spliter_y_new_d,seperators_closeup_n_d=self.find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines) + self.save_plot_of_layout_main(text_regions_p, image_page) + + print("marginals: " + str(time.time() - t1)) + + if not self.full_layout: + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + image_page_rotated_n, textline_mask_tot_d, text_regions_p_1_n = rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, slope_deskew) + + text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1]) + textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1]) + + regions_without_seperators_d = (text_regions_p_1_n[:, :] == 1) * 1 + + regions_without_seperators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_seperators_new(text_regions_p[:,:,0],img_only_regions) + + pixel_lines = 3 + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + num_col, peaks_neg_fin, matrix_of_lines_ch, spliter_y_new, seperators_closeup_n = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + num_col_d, peaks_neg_fin_d, matrix_of_lines_ch_d, spliter_y_new_d, seperators_closeup_n_d = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) K.clear_session() gc.collect() - - #print(peaks_neg_fin,num_col,'num_col2') - - print(num_col_classifier,'num_col_classifier') - - - if num_col_classifier>=3: - if np.abs(slope_deskew)<0.13: - regions_without_seperators=regions_without_seperators.astype(np.uint8) - regions_without_seperators = cv2.erode(regions_without_seperators[:,:], self.kernel, iterations=6) - - random_pixels_for_image=np.random.randn(regions_without_seperators.shape[0],regions_without_seperators.shape[1]) - random_pixels_for_image[random_pixels_for_image<-.5]=0 - random_pixels_for_image[random_pixels_for_image!=0]=1 - - regions_without_seperators[(random_pixels_for_image[:,:]==1) & (text_regions_p[:,:]==2)]=1 - - if np.abs(slope_deskew)>=.13: - regions_without_seperators_d=regions_without_seperators_d.astype(np.uint8) - regions_without_seperators_d = cv2.erode(regions_without_seperators_d[:,:], self.kernel, iterations=6) - - random_pixels_for_image=np.random.randn(regions_without_seperators_d.shape[0],regions_without_seperators_d.shape[1]) - random_pixels_for_image[random_pixels_for_image<-.5]=0 - random_pixels_for_image[random_pixels_for_image!=0]=1 - - regions_without_seperators_d[(random_pixels_for_image[:,:]==1) & (text_regions_p_1_n[:,:]==2)]=1 + + # print(peaks_neg_fin,num_col,'num_col2') + + print(num_col_classifier, "num_col_classifier") + + if num_col_classifier >= 3: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + regions_without_seperators = regions_without_seperators.astype(np.uint8) + regions_without_seperators = cv2.erode(regions_without_seperators[:, :], self.kernel, iterations=6) + + random_pixels_for_image = np.random.randn(regions_without_seperators.shape[0], regions_without_seperators.shape[1]) + random_pixels_for_image[random_pixels_for_image < -0.5] = 0 + random_pixels_for_image[random_pixels_for_image != 0] = 1 + + regions_without_seperators[(random_pixels_for_image[:, :] == 1) & (text_regions_p[:, :] == 2)] = 1 + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + regions_without_seperators_d = regions_without_seperators_d.astype(np.uint8) + regions_without_seperators_d = cv2.erode(regions_without_seperators_d[:, :], self.kernel, iterations=6) + + random_pixels_for_image = np.random.randn(regions_without_seperators_d.shape[0], regions_without_seperators_d.shape[1]) + random_pixels_for_image[random_pixels_for_image < -0.5] = 0 + random_pixels_for_image[random_pixels_for_image != 0] = 1 + + regions_without_seperators_d[(random_pixels_for_image[:, :] == 1) & (text_regions_p_1_n[:, :] == 2)] = 1 else: pass - - if np.abs(slope_deskew)<0.13: - boxes=self.return_boxes_of_images_by_order_of_reading_new(spliter_y_new,regions_without_seperators,matrix_of_lines_ch) + + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + boxes = return_boxes_of_images_by_order_of_reading_new(spliter_y_new, regions_without_seperators, matrix_of_lines_ch) else: - boxes_d=self.return_boxes_of_images_by_order_of_reading_new(spliter_y_new_d,regions_without_seperators_d,matrix_of_lines_ch_d) - - #print(len(boxes),'boxes') - - #sys.exit() - - print('boxes in: '+str(time.time()-t1)) - img_revised_tab=text_regions_p[:,:] - - #plt.imshow(img_revised_tab) - #plt.show() + boxes_d = return_boxes_of_images_by_order_of_reading_new(spliter_y_new_d, regions_without_seperators_d, matrix_of_lines_ch_d) + + # print(len(boxes),'boxes') + + # sys.exit() + + print("boxes in: " + str(time.time() - t1)) + img_revised_tab = text_regions_p[:, :] + + # plt.imshow(img_revised_tab) + # plt.show() K.clear_session() - - - - pixel_img=4 - min_area_mar=0.00001 - polygons_of_marginals=self.return_contours_of_interested_region(text_regions_p,pixel_img,min_area_mar) - + pixel_img = 4 + min_area_mar = 0.00001 + polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar) + + if self.full_layout: + # set first model with second model + text_regions_p[:, :][text_regions_p[:, :] == 2] = 5 + text_regions_p[:, :][text_regions_p[:, :] == 3] = 6 + text_regions_p[:, :][text_regions_p[:, :] == 4] = 8 - - if self.full_layout=='true' or self.full_layout=='True' or self.full_layout=='TRUE': - # set first model with second model - text_regions_p[:,:][text_regions_p[:,:]==2]=5 - text_regions_p[:,:][text_regions_p[:,:]==3]=6 - text_regions_p[:,:][text_regions_p[:,:]==4]=8 - - K.clear_session() - #gc.collect() - - patches=True - - - image_page=image_page.astype(np.uint8) - - #print(type(image_page)) - regions_fully,regions_fully_only_drop=self.extract_text_regions(image_page,patches,cols=num_col_classifier) - - - - - regions_fully_only_drop=self.put_drop_out_from_only_drop_model(regions_fully_only_drop,text_regions_p) - regions_fully[:,:,0][regions_fully_only_drop[:,:,0]==4]=4 + # gc.collect() + + patches = True + + image_page = image_page.astype(np.uint8) + + # print(type(image_page)) + regions_fully, regions_fully_only_drop = self.extract_text_regions(image_page, patches, cols=num_col_classifier) + + regions_fully_only_drop = put_drop_out_from_only_drop_model(regions_fully_only_drop, text_regions_p) + regions_fully[:, :, 0][regions_fully_only_drop[:, :, 0] == 4] = 4 K.clear_session() gc.collect() - - #plt.imshow(regions_fully[:,:,0]) - #plt.show() - - regions_fully=self.putt_bb_of_drop_capitals_of_model_in_patches_in_layout(regions_fully) - - - #plt.imshow(regions_fully[:,:,0]) - #plt.show() + + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + + regions_fully = putt_bb_of_drop_capitals_of_model_in_patches_in_layout(regions_fully) + + # plt.imshow(regions_fully[:,:,0]) + # plt.show() K.clear_session() gc.collect() - patches=False - regions_fully_np,_=self.extract_text_regions(image_page,patches,cols=num_col_classifier) - - - - - - #plt.imshow(regions_fully_np[:,:,0]) - #plt.show() - - if num_col_classifier>2: - regions_fully_np[:,:,0][regions_fully_np[:,:,0]==4]=0 + patches = False + regions_fully_np, _ = self.extract_text_regions(image_page, patches, cols=num_col_classifier) + + # plt.imshow(regions_fully_np[:,:,0]) + # plt.show() + + if num_col_classifier > 2: + regions_fully_np[:, :, 0][regions_fully_np[:, :, 0] == 4] = 0 else: - regions_fully_np=self.filter_small_drop_capitals_from_no_patch_layout(regions_fully_np,text_regions_p) - - #regions_fully_np=self.filter_small_drop_capitals_from_no_patch_layout(regions_fully_np,text_regions_p) - #plt.imshow(regions_fully_np[:,:,0]) - #plt.show() - + regions_fully_np = filter_small_drop_capitals_from_no_patch_layout(regions_fully_np, text_regions_p) + + # regions_fully_np=filter_small_drop_capitals_from_no_patch_layout(regions_fully_np,text_regions_p) + # plt.imshow(regions_fully_np[:,:,0]) + # plt.show() + K.clear_session() gc.collect() - - #plt.imshow(regions_fully[:,:,0]) - #plt.show() - - regions_fully=self.boosting_headers_by_longshot_region_segmentation(regions_fully,regions_fully_np,img_only_regions) - - - #plt.imshow(regions_fully[:,:,0]) - #plt.show() - - text_regions_p[:,:][regions_fully[:,:,0]==4]=4 + + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + + regions_fully = boosting_headers_by_longshot_region_segmentation(regions_fully, regions_fully_np, img_only_regions) + + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + + text_regions_p[:, :][regions_fully[:, :, 0] == 4] = 4 ##text_regions_p[:,:][(regions_fully[:,:,0]==7) & (text_regions_p[:,:]!=0)]=7 - - text_regions_p[:,:][regions_fully_np[:,:,0]==4]=4 - - - #plt.imshow(text_regions_p) - #plt.show() - - - if np.abs(slope_deskew)>=0.13: - image_page_rotated_n,textline_mask_tot_d,text_regions_p_1_n,regions_fully_n=self.rotation_not_90_func_full_layout(image_page,textline_mask_tot,text_regions_p,regions_fully,slope_deskew) - - text_regions_p_1_n=self.resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) - textline_mask_tot_d=self.resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1]) - regions_fully_n=self.resize_image(regions_fully_n,text_regions_p.shape[0],text_regions_p.shape[1]) - - regions_without_seperators_d=(text_regions_p_1_n[:,:]==1)*1 - - - - - - - - regions_without_seperators=(text_regions_p[:,:]==1)*1#( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_seperators_new(text_regions_p[:,:,0],img_only_regions) - - - - - + + text_regions_p[:, :][regions_fully_np[:, :, 0] == 4] = 4 + + # plt.imshow(text_regions_p) + # plt.show() + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + image_page_rotated_n, textline_mask_tot_d, text_regions_p_1_n, regions_fully_n = rotation_not_90_func_full_layout(image_page, textline_mask_tot, text_regions_p, regions_fully, slope_deskew) + + text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1]) + textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1]) + regions_fully_n = resize_image(regions_fully_n, text_regions_p.shape[0], text_regions_p.shape[1]) + + regions_without_seperators_d = (text_regions_p_1_n[:, :] == 1) * 1 + + regions_without_seperators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_seperators_new(text_regions_p[:,:,0],img_only_regions) + K.clear_session() gc.collect() - img_revised_tab=np.copy(text_regions_p[:,:]) - - print('full layout in: '+str(time.time()-t1)) - - #sys.exit() - - - - pixel_img=5 - polygons_of_images=self.return_contours_of_interested_region(img_revised_tab,pixel_img) - - #plt.imshow(img_revised_tab) - #plt.show() - - #print(img_revised_tab.shape,text_regions_p_1_n.shape) - - #text_regions_p_1_n=self.resize_image(text_regions_p_1_n,img_revised_tab.shape[0],img_revised_tab.shape[1]) - - #print(np.unique(text_regions_p_1_n),'uni') - text_only=( (img_revised_tab[:,:]==1) )*1 - if np.abs(slope_deskew)>=0.13: - text_only_d=( (text_regions_p_1_n[:,:]==1) )*1 + img_revised_tab = np.copy(text_regions_p[:, :]) + + print("full layout in: " + str(time.time() - t1)) + + # sys.exit() + + pixel_img = 5 + polygons_of_images = return_contours_of_interested_region(img_revised_tab, pixel_img) + + # plt.imshow(img_revised_tab) + # plt.show() + + # print(img_revised_tab.shape,text_regions_p_1_n.shape) + + # text_regions_p_1_n=resize_image(text_regions_p_1_n,img_revised_tab.shape[0],img_revised_tab.shape[1]) + + # print(np.unique(text_regions_p_1_n),'uni') + text_only = ((img_revised_tab[:, :] == 1)) * 1 + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + text_only_d = ((text_regions_p_1_n[:, :] == 1)) * 1 ##text_only_h=( (img_revised_tab[:,:,0]==2) )*1 - - - - - #print(text_only.shape,text_only_d.shape) - #plt.imshow(text_only) - #plt.show() - - #plt.imshow(text_only_d) - #plt.show() - - min_con_area=0.000005 - - if np.abs(slope_deskew)>=0.13: - - contours_only_text,hir_on_text=self.return_contours_of_image(text_only) - contours_only_text_parent=self.return_parent_contours( contours_only_text,hir_on_text) - - areas_cnt_text=np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) - - - areas_cnt_text=areas_cnt_text/float(text_only.shape[0]*text_only.shape[1]) - - - contours_biggest=contours_only_text_parent[np.argmax(areas_cnt_text)] - contours_only_text_parent=[contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz]>min_con_area] - areas_cnt_text_parent=[areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz]>min_con_area] - - - index_con_parents=np.argsort(areas_cnt_text_parent) - contours_only_text_parent=list(np.array(contours_only_text_parent)[index_con_parents] ) - areas_cnt_text_parent=list(np.array(areas_cnt_text_parent)[index_con_parents] ) - - cx_bigest_big,cy_biggest_big ,_, _, _ ,_,_=self.find_new_features_of_contoures([contours_biggest]) - cx_bigest,cy_biggest ,_, _, _ ,_,_=self.find_new_features_of_contoures(contours_only_text_parent) - - - - contours_only_text_d,hir_on_text_d=self.return_contours_of_image(text_only_d) - contours_only_text_parent_d=self.return_parent_contours( contours_only_text_d,hir_on_text_d) - - - - areas_cnt_text_d=np.array([cv2.contourArea(contours_only_text_parent_d[j]) for j in range(len(contours_only_text_parent_d))]) - - areas_cnt_text_d=areas_cnt_text_d/float(text_only_d.shape[0]*text_only_d.shape[1]) - - - contours_biggest_d=contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] - - cx_bigest_d_big,cy_biggest_d_big ,_, _, _ ,_,_=self.find_new_features_of_contoures([contours_biggest_d]) - cx_bigest_d,cy_biggest_d ,_, _, _ ,_,_=self.find_new_features_of_contoures(contours_only_text_parent_d) - - + + # print(text_only.shape,text_only_d.shape) + # plt.imshow(text_only) + # plt.show() + + # plt.imshow(text_only_d) + # plt.show() + + min_con_area = 0.000005 + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + + contours_only_text, hir_on_text = return_contours_of_image(text_only) + contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) + + areas_cnt_text = np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) + + areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) + + contours_biggest = contours_only_text_parent[np.argmax(areas_cnt_text)] + contours_only_text_parent = [contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz] > min_con_area] + areas_cnt_text_parent = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > min_con_area] + + index_con_parents = np.argsort(areas_cnt_text_parent) + contours_only_text_parent = list(np.array(contours_only_text_parent)[index_con_parents]) + areas_cnt_text_parent = list(np.array(areas_cnt_text_parent)[index_con_parents]) + + cx_bigest_big, cy_biggest_big, _, _, _, _, _ = find_new_features_of_contoures([contours_biggest]) + cx_bigest, cy_biggest, _, _, _, _, _ = find_new_features_of_contoures(contours_only_text_parent) + + contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) + contours_only_text_parent_d = return_parent_contours(contours_only_text_d, hir_on_text_d) + + areas_cnt_text_d = np.array([cv2.contourArea(contours_only_text_parent_d[j]) for j in range(len(contours_only_text_parent_d))]) + + areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) + + contours_biggest_d = contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] + + cx_bigest_d_big, cy_biggest_d_big, _, _, _, _, _ = find_new_features_of_contoures([contours_biggest_d]) + cx_bigest_d, cy_biggest_d, _, _, _, _, _ = find_new_features_of_contoures(contours_only_text_parent_d) + (h, w) = text_only.shape[:2] - center = (w // 2., h // 2.) + center = (w // 2.0, h // 2.0) M = cv2.getRotationMatrix2D(center, slope_deskew, 1.0) - - - M_22=np.array(M)[:2,:2] - - p_big=np.dot(M_22,[cx_bigest_big,cy_biggest_big]) - - x_diff=p_big[0]-cx_bigest_d_big - y_diff=p_big[1]-cy_biggest_d_big - - - #print(p_big) - #print(cx_bigest_d_big,cy_biggest_d_big) - #print(x_diff,y_diff) - - contours_only_text_parent_d_ordered=[] + + M_22 = np.array(M)[:2, :2] + + p_big = np.dot(M_22, [cx_bigest_big, cy_biggest_big]) + + x_diff = p_big[0] - cx_bigest_d_big + y_diff = p_big[1] - cy_biggest_d_big + + # print(p_big) + # print(cx_bigest_d_big,cy_biggest_d_big) + # print(x_diff,y_diff) + + contours_only_text_parent_d_ordered = [] for i in range(len(contours_only_text_parent)): - #img1=np.zeros((text_only.shape[0],text_only.shape[1],3)) - #img1=cv2.fillPoly(img1,pts=[contours_only_text_parent[i]] ,color=(1,1,1)) - - #plt.imshow(img1[:,:,0]) - #plt.show() - - p=np.dot(M_22,[cx_bigest[i],cy_biggest[i]]) - #print(p) - p[0]=p[0]-x_diff[0] - p[1]=p[1]-y_diff[0] - #print(p) - #print(cx_bigest_d) - #print(cy_biggest_d) - dists = [math.sqrt((p[0]-cx_bigest_d[j])**2 + (p[1]-cy_biggest_d[j])**2) for j in range(len(cx_bigest_d))] - #print(np.argmin(dists)) + # img1=np.zeros((text_only.shape[0],text_only.shape[1],3)) + # img1=cv2.fillPoly(img1,pts=[contours_only_text_parent[i]] ,color=(1,1,1)) + + # plt.imshow(img1[:,:,0]) + # plt.show() + + p = np.dot(M_22, [cx_bigest[i], cy_biggest[i]]) + # print(p) + p[0] = p[0] - x_diff[0] + p[1] = p[1] - y_diff[0] + # print(p) + # print(cx_bigest_d) + # print(cy_biggest_d) + dists = [math.sqrt((p[0] - cx_bigest_d[j]) ** 2 + (p[1] - cy_biggest_d[j]) ** 2) for j in range(len(cx_bigest_d))] + # print(np.argmin(dists)) contours_only_text_parent_d_ordered.append(contours_only_text_parent_d[np.argmin(dists)]) - - #img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) - #img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d[np.argmin(dists)]] ,color=(1,1,1)) - - #plt.imshow(img2[:,:,0]) - #plt.show() + + # img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) + # img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d[np.argmin(dists)]] ,color=(1,1,1)) + + # plt.imshow(img2[:,:,0]) + # plt.show() else: - contours_only_text,hir_on_text=self.return_contours_of_image(text_only) - contours_only_text_parent=self.return_parent_contours( contours_only_text,hir_on_text) - - areas_cnt_text=np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) - - - areas_cnt_text=areas_cnt_text/float(text_only.shape[0]*text_only.shape[1]) - - - contours_biggest=contours_only_text_parent[np.argmax(areas_cnt_text)] - contours_only_text_parent=[contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz]>min_con_area] - areas_cnt_text_parent=[areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz]>min_con_area] - - - index_con_parents=np.argsort(areas_cnt_text_parent) - contours_only_text_parent=list(np.array(contours_only_text_parent)[index_con_parents] ) - areas_cnt_text_parent=list(np.array(areas_cnt_text_parent)[index_con_parents] ) - - cx_bigest_big,cy_biggest_big ,_, _, _ ,_,_=self.find_new_features_of_contoures([contours_biggest]) - cx_bigest,cy_biggest ,_, _, _ ,_,_=self.find_new_features_of_contoures(contours_only_text_parent) - #print(areas_cnt_text_parent,'areas_cnt_text_parent') - + contours_only_text, hir_on_text = return_contours_of_image(text_only) + contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) + + areas_cnt_text = np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) + + areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) + + contours_biggest = contours_only_text_parent[np.argmax(areas_cnt_text)] + contours_only_text_parent = [contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz] > min_con_area] + areas_cnt_text_parent = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > min_con_area] + + index_con_parents = np.argsort(areas_cnt_text_parent) + contours_only_text_parent = list(np.array(contours_only_text_parent)[index_con_parents]) + areas_cnt_text_parent = list(np.array(areas_cnt_text_parent)[index_con_parents]) + + cx_bigest_big, cy_biggest_big, _, _, _, _, _ = find_new_features_of_contoures([contours_biggest]) + cx_bigest, cy_biggest, _, _, _, _, _ = find_new_features_of_contoures(contours_only_text_parent) + # print(areas_cnt_text_parent,'areas_cnt_text_parent') + ###index_con_parents_d=np.argsort(areas_cnt_text_parent_d) ##contours_only_text_parent_d=list(np.array(contours_only_text_parent_d)[index_con_parents_d]) ###areas_cnt_text_parent_d=list(np.array(areas_cnt_text_parent_d)[index_con_parents_d]) - + ##print(areas_cnt_text_parent_d,'areas_cnt_text_parent_d') - - #print(len(contours_only_text_parent),len(contours_only_text_parent_d),'vizzz') - - - - - - - - - - + # print(len(contours_only_text_parent),len(contours_only_text_parent_d),'vizzz') - - txt_con_org=self.get_textregion_contours_in_org_image(contours_only_text_parent,self.image,slope_first) + txt_con_org = get_textregion_contours_in_org_image(contours_only_text_parent, self.image, slope_first) - ###boxes_text,_=self.get_text_region_boxes_by_given_contours(contours_only_text_parent) - boxes_text,_=self.get_text_region_boxes_by_given_contours(contours_only_text_parent) - boxes_marginals,_=self.get_text_region_boxes_by_given_contours(polygons_of_marginals) - ####boxes_text_h,_=self.get_text_region_boxes_by_given_contours(text_only_h,contours_only_text_parent_h,image_page) - + ###boxes_text,_= get_text_region_boxes_by_given_contours(contours_only_text_parent) + boxes_text, _ = get_text_region_boxes_by_given_contours(contours_only_text_parent) + boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) + ####boxes_text_h,_= get_text_region_boxes_by_given_contours(text_only_h,contours_only_text_parent_h,image_page) - + if not self.curved_line: + slopes, all_found_texline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, index_by_text_par_con = self.get_slopes_and_deskew_new(txt_con_org, contours_only_text_parent, textline_mask_tot_ea, image_page_rotated, boxes_text, slope_deskew) + + slopes_marginals, all_found_texline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, all_box_coord_marginals, index_by_text_par_con_marginal = self.get_slopes_and_deskew_new(polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, image_page_rotated, boxes_marginals, slope_deskew) + + if self.curved_line: + scale_param = 1 + all_found_texline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_curved(txt_con_org, contours_only_text_parent, cv2.erode(textline_mask_tot_ea, kernel=self.kernel, iterations=1), image_page_rotated, boxes_text, text_only, num_col_classifier, scale_param, slope_deskew) + + # all_found_texline_polygons,boxes_text,txt_con_org,contours_only_text_parent,all_box_coord=self.get_slopes_and_deskew_new_curved(txt_con_org,contours_only_text_parent,textline_mask_tot_ea,image_page_rotated,boxes_text,text_only,num_col,scale_param) + all_found_texline_polygons = small_textlines_to_parent_adherence2(all_found_texline_polygons, textline_mask_tot_ea, num_col_classifier) + + # slopes=list(np.zeros(len(contours_only_text_parent))) + + all_found_texline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, all_box_coord_marginals, index_by_text_par_con_marginal, slopes_marginals = self.get_slopes_and_deskew_new_curved(polygons_of_marginals, polygons_of_marginals, cv2.erode(textline_mask_tot_ea, kernel=self.kernel, iterations=1), image_page_rotated, boxes_marginals, text_only, num_col_classifier, scale_param, slope_deskew) + + # all_found_texline_polygons,boxes_text,txt_con_org,contours_only_text_parent,all_box_coord=self.get_slopes_and_deskew_new_curved(txt_con_org,contours_only_text_parent,textline_mask_tot_ea,image_page_rotated,boxes_text,text_only,num_col,scale_param) + all_found_texline_polygons_marginals = small_textlines_to_parent_adherence2(all_found_texline_polygons_marginals, textline_mask_tot_ea, num_col_classifier) + + index_of_vertical_text_contours = np.array(range(len(slopes)))[(abs(np.array(slopes)) > 60)] + + contours_text_vertical = [contours_only_text_parent[i] for i in index_of_vertical_text_contours] - if self.curved_line is None or self.curved_line=='false' or self.curved_line=='False' or self.curved_line=='FALSE' : - slopes,all_found_texline_polygons,boxes_text,txt_con_org,contours_only_text_parent,all_box_coord,index_by_text_par_con=self.get_slopes_and_deskew_new(txt_con_org,contours_only_text_parent,textline_mask_tot_ea,image_page_rotated,boxes_text,slope_deskew) - - - slopes_marginals,all_found_texline_polygons_marginals,boxes_marginals,_,polygons_of_marginals,all_box_coord_marginals,index_by_text_par_con_marginal=self.get_slopes_and_deskew_new(polygons_of_marginals,polygons_of_marginals,textline_mask_tot_ea,image_page_rotated,boxes_marginals,slope_deskew) - - - if self.curved_line=='true' or self.curved_line=='True' or self.curved_line=='TRUE' : - scale_param=1 - all_found_texline_polygons,boxes_text,txt_con_org,contours_only_text_parent,all_box_coord,index_by_text_par_con,slopes=self.get_slopes_and_deskew_new_curved(txt_con_org,contours_only_text_parent,cv2.erode(textline_mask_tot_ea,kernel=self.kernel,iterations=1),image_page_rotated,boxes_text,text_only,num_col_classifier,scale_param,slope_deskew) - - #all_found_texline_polygons,boxes_text,txt_con_org,contours_only_text_parent,all_box_coord=self.get_slopes_and_deskew_new_curved(txt_con_org,contours_only_text_parent,textline_mask_tot_ea,image_page_rotated,boxes_text,text_only,num_col,scale_param) - all_found_texline_polygons=self.small_textlines_to_parent_adherence2(all_found_texline_polygons,textline_mask_tot_ea,num_col_classifier) - - #slopes=list(np.zeros(len(contours_only_text_parent))) - - - all_found_texline_polygons_marginals,boxes_marginals,_,polygons_of_marginals,all_box_coord_marginals,index_by_text_par_con_marginal,slopes_marginals=self.get_slopes_and_deskew_new_curved(polygons_of_marginals,polygons_of_marginals,cv2.erode(textline_mask_tot_ea,kernel=self.kernel,iterations=1),image_page_rotated,boxes_marginals,text_only,num_col_classifier,scale_param,slope_deskew) - - #all_found_texline_polygons,boxes_text,txt_con_org,contours_only_text_parent,all_box_coord=self.get_slopes_and_deskew_new_curved(txt_con_org,contours_only_text_parent,textline_mask_tot_ea,image_page_rotated,boxes_text,text_only,num_col,scale_param) - all_found_texline_polygons_marginals=self.small_textlines_to_parent_adherence2(all_found_texline_polygons_marginals,textline_mask_tot_ea,num_col_classifier) - - index_of_vertical_text_contours=np.array(range(len(slopes)) )[(abs(np.array(slopes))>60)] - - contours_text_vertical=[contours_only_text_parent[i] for i in index_of_vertical_text_contours] - K.clear_session() gc.collect() - - - #contours_only_text_parent_d_ordered=list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) + + # contours_only_text_parent_d_ordered=list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) ###print(index_by_text_par_con,'index_by_text_par_con') - - if self.full_layout=='true' or self.full_layout=='True' or self.full_layout=='TRUE': + + if self.full_layout: ##for iii in range(len(contours_only_text_parent)): - ##img1=np.zeros((text_only.shape[0],text_only.shape[1],3)) - ##img1=cv2.fillPoly(img1,pts=[contours_only_text_parent[iii]] ,color=(1,1,1)) - - ##plt.imshow(img1[:,:,0]) - ##plt.show() - - ##img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) - ##img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d_ordered[iii]] ,color=(1,1,1)) - - ##plt.imshow(img2[:,:,0]) - ##plt.show() - - if np.abs(slope_deskew)>=0.13: - contours_only_text_parent_d_ordered=list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) - - - text_regions_p,contours_only_text_parent,contours_only_text_parent_h,all_box_coord,all_box_coord_h,all_found_texline_polygons,all_found_texline_polygons_h,slopes,slopes_h,contours_only_text_parent_d_ordered,contours_only_text_parent_h_d_ordered=self.check_any_text_region_in_model_one_is_main_or_header(text_regions_p,regions_fully,contours_only_text_parent,all_box_coord,all_found_texline_polygons,slopes,contours_only_text_parent_d_ordered) + ##img1=np.zeros((text_only.shape[0],text_only.shape[1],3)) + ##img1=cv2.fillPoly(img1,pts=[contours_only_text_parent[iii]] ,color=(1,1,1)) + + ##plt.imshow(img1[:,:,0]) + ##plt.show() + + ##img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) + ##img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d_ordered[iii]] ,color=(1,1,1)) + + ##plt.imshow(img2[:,:,0]) + ##plt.show() + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + contours_only_text_parent_d_ordered = list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) + + text_regions_p, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, slopes, slopes_h, contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered = check_any_text_region_in_model_one_is_main_or_header(text_regions_p, regions_fully, contours_only_text_parent, all_box_coord, all_found_texline_polygons, slopes, contours_only_text_parent_d_ordered) else: - contours_only_text_parent_d_ordered=None - - - text_regions_p,contours_only_text_parent,contours_only_text_parent_h,all_box_coord,all_box_coord_h,all_found_texline_polygons,all_found_texline_polygons_h,slopes,slopes_h,contours_only_text_parent_d_ordered,contours_only_text_parent_h_d_ordered=self.check_any_text_region_in_model_one_is_main_or_header(text_regions_p,regions_fully,contours_only_text_parent,all_box_coord,all_found_texline_polygons,slopes,contours_only_text_parent_d_ordered) - - - ###text_regions_p,contours_only_text_parent,contours_only_text_parent_h,all_box_coord,all_box_coord_h,all_found_texline_polygons,all_found_texline_polygons_h=self.check_any_text_region_in_model_one_is_main_or_header(text_regions_p,regions_fully,contours_only_text_parent,all_box_coord,all_found_texline_polygons) - #text_regions_p=self.return_region_segmentation_after_implementing_not_head_maintext_parallel(text_regions_p,boxes) - + contours_only_text_parent_d_ordered = None + + text_regions_p, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, slopes, slopes_h, contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered = check_any_text_region_in_model_one_is_main_or_header(text_regions_p, regions_fully, contours_only_text_parent, all_box_coord, all_found_texline_polygons, slopes, contours_only_text_parent_d_ordered) + + ###text_regions_p,contours_only_text_parent,contours_only_text_parent_h,all_box_coord,all_box_coord_h,all_found_texline_polygons,all_found_texline_polygons_h=check_any_text_region_in_model_one_is_main_or_header(text_regions_p,regions_fully,contours_only_text_parent,all_box_coord,all_found_texline_polygons) + # text_regions_p=self.return_region_segmentation_after_implementing_not_head_maintext_parallel(text_regions_p,boxes) + # if you want to save the layout result just uncommet following plot - - + if self.dir_of_layout is not None: - self.save_plot_of_layout(text_regions_p,image_page) + self.save_plot_of_layout(text_regions_p, image_page) if self.dir_of_all is not None: - self.save_plot_of_layout_all(text_regions_p,image_page) + self.save_plot_of_layout_all(text_regions_p, image_page) - - K.clear_session() gc.collect() - + ##print('Job done in: '+str(time.time()-t1)) - - - polygons_of_tabels=[] - - - pixel_img=4 - polygons_of_drop_capitals=self.return_contours_of_interested_region_by_min_size(text_regions_p,pixel_img) - #polygons_of_drop_capitals=[] - - all_found_texline_polygons=self.adhere_drop_capital_region_into_cprresponding_textline(text_regions_p,polygons_of_drop_capitals,contours_only_text_parent,contours_only_text_parent_h,all_box_coord,all_box_coord_h,all_found_texline_polygons,all_found_texline_polygons_h) - - - #print(len(contours_only_text_parent_h),len(contours_only_text_parent_h_d_ordered),'contours_only_text_parent_h') - pixel_lines=6 - - if self.headers_off is None or self.headers_off=='false' or self.headers_off=='False' or self.headers_off=='FALSE': - if np.abs(slope_deskew)<0.13: - num_col,peaks_neg_fin,matrix_of_lines_ch,spliter_y_new,seperators_closeup_n=self.find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines,contours_only_text_parent_h) + + polygons_of_tabels = [] + + pixel_img = 4 + polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img) + # polygons_of_drop_capitals=[] + + all_found_texline_polygons = adhere_drop_capital_region_into_cprresponding_textline(text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, kernel=self.kernel, curved_line=self.curved_line) + + # print(len(contours_only_text_parent_h),len(contours_only_text_parent_h_d_ordered),'contours_only_text_parent_h') + pixel_lines = 6 + + if not self.headers_off: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + num_col, peaks_neg_fin, matrix_of_lines_ch, spliter_y_new, seperators_closeup_n = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines, contours_only_text_parent_h) else: - num_col_d,peaks_neg_fin_d,matrix_of_lines_ch_d,spliter_y_new_d,seperators_closeup_n_d=self.find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines,contours_only_text_parent_h_d_ordered) - elif self.headers_off=='true' or self.headers_off=='True' or self.headers_off=='TRUE': - if np.abs(slope_deskew)<0.13: - num_col,peaks_neg_fin,matrix_of_lines_ch,spliter_y_new,seperators_closeup_n=self.find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines) + num_col_d, peaks_neg_fin_d, matrix_of_lines_ch_d, spliter_y_new_d, seperators_closeup_n_d = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines, contours_only_text_parent_h_d_ordered) + elif self.headers_off: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + num_col, peaks_neg_fin, matrix_of_lines_ch, spliter_y_new, seperators_closeup_n = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) else: - num_col_d,peaks_neg_fin_d,matrix_of_lines_ch_d,spliter_y_new_d,seperators_closeup_n_d=self.find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines) - - #print(peaks_neg_fin,peaks_neg_fin_d,'num_col2') - - #print(spliter_y_new,spliter_y_new_d,'num_col_classifier') - - #print(matrix_of_lines_ch.shape,matrix_of_lines_ch_d.shape,'matrix_of_lines_ch') - - - - if num_col_classifier>=3: - - if np.abs(slope_deskew)<0.13: - regions_without_seperators=regions_without_seperators.astype(np.uint8) - regions_without_seperators = cv2.erode(regions_without_seperators[:,:], self.kernel, iterations=6) - - random_pixels_for_image=np.random.randn(regions_without_seperators.shape[0],regions_without_seperators.shape[1]) - random_pixels_for_image[random_pixels_for_image<-.5]=0 - random_pixels_for_image[random_pixels_for_image!=0]=1 - - regions_without_seperators[(random_pixels_for_image[:,:]==1) & (text_regions_p[:,:]==5)]=1 - + num_col_d, peaks_neg_fin_d, matrix_of_lines_ch_d, spliter_y_new_d, seperators_closeup_n_d = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) + + # print(peaks_neg_fin,peaks_neg_fin_d,'num_col2') + + # print(spliter_y_new,spliter_y_new_d,'num_col_classifier') + + # print(matrix_of_lines_ch.shape,matrix_of_lines_ch_d.shape,'matrix_of_lines_ch') + + if num_col_classifier >= 3: + + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + regions_without_seperators = regions_without_seperators.astype(np.uint8) + regions_without_seperators = cv2.erode(regions_without_seperators[:, :], self.kernel, iterations=6) + + random_pixels_for_image = np.random.randn(regions_without_seperators.shape[0], regions_without_seperators.shape[1]) + random_pixels_for_image[random_pixels_for_image < -0.5] = 0 + random_pixels_for_image[random_pixels_for_image != 0] = 1 + + regions_without_seperators[(random_pixels_for_image[:, :] == 1) & (text_regions_p[:, :] == 5)] = 1 + else: - - - regions_without_seperators_d=regions_without_seperators_d.astype(np.uint8) - regions_without_seperators_d = cv2.erode(regions_without_seperators_d[:,:], self.kernel, iterations=6) - - - random_pixels_for_image=np.random.randn(regions_without_seperators_d.shape[0],regions_without_seperators_d.shape[1]) - random_pixels_for_image[random_pixels_for_image<-.5]=0 - random_pixels_for_image[random_pixels_for_image!=0]=1 - - - - regions_without_seperators_d[(random_pixels_for_image[:,:]==1) & (text_regions_p_1_n[:,:]==5)]=1 + + regions_without_seperators_d = regions_without_seperators_d.astype(np.uint8) + regions_without_seperators_d = cv2.erode(regions_without_seperators_d[:, :], self.kernel, iterations=6) + + random_pixels_for_image = np.random.randn(regions_without_seperators_d.shape[0], regions_without_seperators_d.shape[1]) + random_pixels_for_image[random_pixels_for_image < -0.5] = 0 + random_pixels_for_image[random_pixels_for_image != 0] = 1 + + regions_without_seperators_d[(random_pixels_for_image[:, :] == 1) & (text_regions_p_1_n[:, :] == 5)] = 1 else: pass - - - if np.abs(slope_deskew)<0.13: - boxes=self.return_boxes_of_images_by_order_of_reading_new(spliter_y_new,regions_without_seperators,matrix_of_lines_ch) + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + boxes = return_boxes_of_images_by_order_of_reading_new(spliter_y_new, regions_without_seperators, matrix_of_lines_ch) else: - boxes_d=self.return_boxes_of_images_by_order_of_reading_new(spliter_y_new_d,regions_without_seperators_d,matrix_of_lines_ch_d) - - #print(boxes_d,len(boxes_d),'boxes_d') - #for mv in range(len(boxes_d)): - - #img_box=image_page[boxes_d[mv][1]:boxes_d[mv][1]+boxes_d[mv][3] , boxes_d[mv][0]:boxes_d[mv][0]+boxes_d[mv][2],: ] - #plt.imshow(img_box) - #plt.show() - + boxes_d = return_boxes_of_images_by_order_of_reading_new(spliter_y_new_d, regions_without_seperators_d, matrix_of_lines_ch_d) - #print(slopes) + # print(slopes) if self.dir_of_cropped_images is not None: - self.write_images_into_directory(polygons_of_images,self.dir_of_cropped_images,image_page) - + self.write_images_into_directory(polygons_of_images, self.dir_of_cropped_images, image_page) - if self.full_layout=='true' or self.full_layout=='True' or self.full_layout=='TRUE': - if np.abs(slope_deskew)<0.13: - order_text_new , id_of_texts_tot=self.do_order_of_regions(contours_only_text_parent,contours_only_text_parent_h,boxes,textline_mask_tot) + if self.full_layout: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) else: - order_text_new , id_of_texts_tot=self.do_order_of_regions(contours_only_text_parent_d_ordered,contours_only_text_parent_h_d_ordered,boxes_d,textline_mask_tot_d) + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) - - - - self.write_into_page_xml_full(contours_only_text_parent,contours_only_text_parent_h,page_coord,self.dir_out , order_text_new , id_of_texts_tot,all_found_texline_polygons,all_found_texline_polygons_h, - all_box_coord,all_box_coord_h,polygons_of_images ,polygons_of_tabels,polygons_of_drop_capitals,polygons_of_marginals,all_found_texline_polygons_marginals,all_box_coord_marginals,slopes,slopes_marginals) + self.write_into_page_xml_full(contours_only_text_parent, contours_only_text_parent_h, page_coord, self.dir_out, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_found_texline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, polygons_of_tabels, polygons_of_drop_capitals, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals) else: - contours_only_text_parent_h=None - #print('bura galmir?') - if np.abs(slope_deskew)<0.13: - contours_only_text_parent=list(np.array(contours_only_text_parent)[index_by_text_par_con]) - order_text_new , id_of_texts_tot=self.do_order_of_regions(contours_only_text_parent,contours_only_text_parent_h,boxes,textline_mask_tot) + contours_only_text_parent_h = None + # print('bura galmir?') + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + contours_only_text_parent = list(np.array(contours_only_text_parent)[index_by_text_par_con]) + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) else: - contours_only_text_parent_d_ordered=list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) - order_text_new , id_of_texts_tot=self.do_order_of_regions(contours_only_text_parent_d_ordered,contours_only_text_parent_h,boxes_d,textline_mask_tot_d) - #order_text_new , id_of_texts_tot=self.do_order_of_regions(contours_only_text_parent,contours_only_text_parent_h,boxes,textline_mask_tot) - self.write_into_page_xml(txt_con_org,page_coord,self.dir_out , order_text_new , id_of_texts_tot,all_found_texline_polygons, - all_box_coord,polygons_of_images,polygons_of_marginals,all_found_texline_polygons_marginals,all_box_coord_marginals,self.curved_line,slopes,slopes_marginals) - - except: - txt_con_org=[] - order_text_new=[] - id_of_texts_tot=[] - all_found_texline_polygons=[] - all_box_coord=[] - polygons_of_images=[] - polygons_of_marginals=[] - all_found_texline_polygons_marginals=[] - all_box_coord_marginals=[] - slopes=[] - slopes_marginals=[] - self.write_into_page_xml(txt_con_org,page_coord,self.dir_out , order_text_new , id_of_texts_tot,all_found_texline_polygons, - all_box_coord,polygons_of_images,polygons_of_marginals,all_found_texline_polygons_marginals,all_box_coord_marginals,self.curved_line,slopes,slopes_marginals) - - - print('Job done in: '+str(time.time()-t1)) - - + contours_only_text_parent_d_ordered = list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) + # order_text_new , id_of_texts_tot=self.do_order_of_regions(contours_only_text_parent,contours_only_text_parent_h,boxes,textline_mask_tot) + self.write_into_page_xml(txt_con_org, page_coord, self.dir_out, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, self.curved_line, slopes, slopes_marginals) - + except: + txt_con_org = [] + order_text_new = [] + id_of_texts_tot = [] + all_found_texline_polygons = [] + all_box_coord = [] + polygons_of_images = [] + polygons_of_marginals = [] + all_found_texline_polygons_marginals = [] + all_box_coord_marginals = [] + slopes = [] + slopes_marginals = [] + self.write_into_page_xml(txt_con_org, page_coord, self.dir_out, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, self.curved_line, slopes, slopes_marginals) + + print("Job done in: " + str(time.time() - t1)) -@click.command() -@click.option('--image', '-i', help='image filename', type=click.Path(exists=True, dir_okay=False)) -@click.option('--out', '-o', help='directory to write output xml data', type=click.Path(exists=True, file_okay=False)) -@click.option('--model', '-m', help='directory of models', type=click.Path(exists=True, file_okay=False)) -@click.option('--save_images', '-si', help='if a directory is given, images in documents will be cropped and saved there', type=click.Path(exists=True, file_okay=False)) -@click.option('--save_layout', '-sl', help='if a directory is given, plot of layout will be saved there', type=click.Path(exists=True, file_okay=False)) -@click.option('--save_deskewed', '-sd', help='if a directory is given, deskewed image will be saved there', type=click.Path(exists=True, file_okay=False)) -@click.option('--save_all', '-sa', help='if a directory is given, all plots needed for documentation will be saved there', type=click.Path(exists=True, file_okay=False)) -@click.option('--allow_enhancement', '-ae', help='if this parameter set to true, this tool would check that input image need resizing and enhancement or not. If so output of resized and enhanced image and corresponding layout data will be written in out directory') -@click.option('--curved_line', '-cl', help='if this parameter set to true, this tool will try to return contoure of textlines instead of rectabgle bounding box of textline. This should be taken into account that with this option the tool need more time to do process.') - -@click.option('--full_layout', '-fl', help='if this parameter set to true, this tool will try to return all elements of layout.') -@click.option('--allow_scaling', '-as', help='if this parameter set to true, this tool would check the scale and if needed it will scale it to perform better layout detection') -@click.option('--headers_off', '-ho', help='if this parameter set to true, this tool would ignore headers role in reading order') -def main(image,out, model,save_images,save_layout,save_deskewed,save_all,allow_enhancement,curved_line,full_layout,allow_scaling,headers_off): - possibles = globals() # XXX unused? - possibles.update(locals()) - x = eynollah(image, None, out, model,save_images,save_layout,save_deskewed,save_all,allow_enhancement,curved_line,full_layout,allow_scaling,headers_off) - x.run() - - -if __name__ == "__main__": - main() - diff --git a/sbb_newspapers_org_image/unused.py b/sbb_newspapers_org_image/unused.py new file mode 100644 index 0000000..45bfdd7 --- /dev/null +++ b/sbb_newspapers_org_image/unused.py @@ -0,0 +1,3029 @@ +""" +Unused methods from eynollah +""" + +import numpy as np +from shapely import geometry +import cv2 + +def color_images_diva(seg, n_classes): + """ + XXX unused + """ + ann_u = range(n_classes) + if len(np.shape(seg)) == 3: + seg = seg[:, :, 0] + + seg_img = np.zeros((np.shape(seg)[0], np.shape(seg)[1], 3)).astype(float) + # colors=sns.color_palette("hls", n_classes) + colors = [[1, 0, 0], [8, 0, 0], [2, 0, 0], [4, 0, 0]] + + for c in ann_u: + c = int(c) + segl = seg == c + seg_img[:, :, 0][seg == c] = colors[c][0] # segl*(colors[c][0]) + seg_img[:, :, 1][seg == c] = colors[c][1] # seg_img[:,:,1]=segl*(colors[c][1]) + seg_img[:, :, 2][seg == c] = colors[c][2] # seg_img[:,:,2]=segl*(colors[c][2]) + return seg_img + +def find_polygons_size_filter(contours, median_area, scaler_up=1.2, scaler_down=0.8): + """ + XXX unused + """ + found_polygons_early = list() + + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + area = polygon.area + # Check that polygon has area greater than minimal area + if area >= median_area * scaler_down and area <= median_area * scaler_up: + found_polygons_early.append(np.array([point for point in polygon.exterior.coords], dtype=np.uint)) + return found_polygons_early + +def resize_ann(seg_in, input_height, input_width): + """ + XXX unused + """ + return cv2.resize(seg_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) + +def get_one_hot(seg, input_height, input_width, n_classes): + seg = seg[:, :, 0] + seg_f = np.zeros((input_height, input_width, n_classes)) + for j in range(n_classes): + seg_f[:, :, j] = (seg == j).astype(int) + return seg_f + +def color_images(seg, n_classes): + ann_u = range(n_classes) + if len(np.shape(seg)) == 3: + seg = seg[:, :, 0] + + seg_img = np.zeros((np.shape(seg)[0], np.shape(seg)[1], 3)).astype(np.uint8) + colors = sns.color_palette("hls", n_classes) + + for c in ann_u: + c = int(c) + segl = seg == c + seg_img[:, :, 0] = segl * c + seg_img[:, :, 1] = segl * c + seg_img[:, :, 2] = segl * c + return seg_img + +def cleaning_probs(probs, sigma): + # Smooth + if sigma > 0.0: + return cv2.GaussianBlur(probs, (int(3 * sigma) * 2 + 1, int(3 * sigma) * 2 + 1), sigma) + elif sigma == 0.0: + return cv2.fastNlMeansDenoising((probs * 255).astype(np.uint8), h=20) / 255 + else: # Negative sigma, do not do anything + return probs + + +def early_deskewing_slope_calculation_based_on_lines(region_pre_p): + # lines are labels by 6 in this model + seperators_closeup = ((region_pre_p[:, :, :] == 6)) * 1 + + seperators_closeup = seperators_closeup.astype(np.uint8) + imgray = cv2.cvtColor(seperators_closeup, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_lines, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = find_features_of_lines(contours_lines) + + slope_lines_org_hor = slope_lines_org[slope_lines == 0] + args = np.array(range(len(slope_lines))) + len_x = seperators_closeup.shape[1] / 4.0 + + args_hor = args[slope_lines == 0] + dist_x_hor = dist_x[slope_lines == 0] + x_min_main_hor = x_min_main[slope_lines == 0] + x_max_main_hor = x_max_main[slope_lines == 0] + cy_main_hor = cy_main[slope_lines == 0] + + args_hor = args_hor[dist_x_hor >= len_x / 2.0] + x_max_main_hor = x_max_main_hor[dist_x_hor >= len_x / 2.0] + x_min_main_hor = x_min_main_hor[dist_x_hor >= len_x / 2.0] + cy_main_hor = cy_main_hor[dist_x_hor >= len_x / 2.0] + slope_lines_org_hor = slope_lines_org_hor[dist_x_hor >= len_x / 2.0] + + slope_lines_org_hor = slope_lines_org_hor[np.abs(slope_lines_org_hor) < 1.2] + slope_mean_hor = np.mean(slope_lines_org_hor) + + if np.abs(slope_mean_hor) > 1.2: + slope_mean_hor = 0 + + # deskewed_new=rotate_image(image_regions_eraly_p[:,:,:],slope_mean_hor) + + args_ver = args[slope_lines == 1] + y_min_main_ver = y_min_main[slope_lines == 1] + y_max_main_ver = y_max_main[slope_lines == 1] + x_min_main_ver = x_min_main[slope_lines == 1] + x_max_main_ver = x_max_main[slope_lines == 1] + cx_main_ver = cx_main[slope_lines == 1] + dist_y_ver = y_max_main_ver - y_min_main_ver + len_y = seperators_closeup.shape[0] / 3.0 + + return slope_mean_hor, cx_main_ver, dist_y_ver + +def boosting_text_only_regions_by_header(textregion_pre_np, img_only_text): + result = ((img_only_text[:, :] == 1) | (textregion_pre_np[:, :, 0] == 2)) * 1 + return result + +def return_rotated_contours(slope, img_patch): + dst = rotate_image(img_patch, slope) + dst = dst.astype(np.uint8) + dst = dst[:, :, 0] + dst[dst != 0] = 1 + + imgray = cv2.cvtColor(dst, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel) + thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, kernel) + contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + return contours + +def get_textlines_for_each_textregions(self, textline_mask_tot, boxes): + textline_mask_tot = cv2.erode(textline_mask_tot, self.kernel, iterations=1) + self.area_of_cropped = [] + self.all_text_region_raw = [] + for jk in range(len(boxes)): + crop_img, crop_coor = crop_image_inside_box(boxes[jk], np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) + crop_img = crop_img.astype(np.uint8) + self.all_text_region_raw.append(crop_img[:, :, 0]) + self.area_of_cropped.append(crop_img.shape[0] * crop_img.shape[1]) + +def deskew_region_prediction(regions_prediction, slope): + image_regions_deskewd = np.zeros(regions_prediction[:, :].shape) + for ind in np.unique(regions_prediction[:, :]): + interest_reg = (regions_prediction[:, :] == ind) * 1 + interest_reg = interest_reg.astype(np.uint8) + deskewed_new = rotate_image(interest_reg, slope) + deskewed_new = deskewed_new[:, :] + deskewed_new[deskewed_new != 0] = ind + + image_regions_deskewd = image_regions_deskewd + deskewed_new + return image_regions_deskewd + +def deskew_erarly(textline_mask): + textline_mask_org = np.copy(textline_mask) + # print(textline_mask.shape,np.unique(textline_mask),'hizzzzz') + # slope_new=0#deskew_images(img_patch) + + textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 + + textline_mask = textline_mask.astype(np.uint8) + kernel = np.ones((5, 5), np.uint8) + + imgray = cv2.cvtColor(textline_mask, cv2.COLOR_BGR2GRAY) + + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours, hirarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(hirarchy) + + commenst_contours = filter_contours_area_of_image(thresh, contours, hirarchy, max_area=0.01, min_area=0.003) + main_contours = filter_contours_area_of_image(thresh, contours, hirarchy, max_area=1, min_area=0.003) + interior_contours = filter_contours_area_of_image_interiors(thresh, contours, hirarchy, max_area=1, min_area=0) + + img_comm = np.zeros(thresh.shape) + img_comm_in = cv2.fillPoly(img_comm, pts=main_contours, color=(255, 255, 255)) + ###img_comm_in=cv2.fillPoly(img_comm, pts =interior_contours, color=(0,0,0)) + + img_comm_in = np.repeat(img_comm_in[:, :, np.newaxis], 3, axis=2) + img_comm_in = img_comm_in.astype(np.uint8) + + imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY) + ##imgray = cv2.cvtColor(image, cv2.COLOR_BGR2HSV) + + ##mask = cv2.inRange(imgray, lower_blue, upper_blue) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + # print(np.unique(mask)) + ##ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + ##plt.imshow(thresh) + ##plt.show() + + contours, hirarchy = cv2.findContours(thresh.copy(), cv2.cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + areas = [cv2.contourArea(contours[jj]) for jj in range(len(contours))] + + median_area = np.mean(areas) + contours_slope = contours # self.find_polugons_size_filter(contours,median_area=median_area,scaler_up=100,scaler_down=0.5) + + if len(contours_slope) > 0: + for jv in range(len(contours_slope)): + new_poly = list(contours_slope[jv]) + if jv == 0: + merged_all = new_poly + else: + merged_all = merged_all + new_poly + + merge = np.array(merged_all) + + img_in = np.zeros(textline_mask.shape) + img_p_in = cv2.fillPoly(img_in, pts=[merge], color=(255, 255, 255)) + + ##plt.imshow(img_p_in) + ##plt.show() + + rect = cv2.minAreaRect(merge) + + box = cv2.boxPoints(rect) + + box = np.int0(box) + + indexes = [0, 1, 2, 3] + x_list = box[:, 0] + y_list = box[:, 1] + + index_y_sort = np.argsort(y_list) + + index_upper_left = index_y_sort[np.argmin(x_list[index_y_sort[0:2]])] + index_upper_right = index_y_sort[np.argmax(x_list[index_y_sort[0:2]])] + + index_lower_left = index_y_sort[np.argmin(x_list[index_y_sort[2:]]) + 2] + index_lower_right = index_y_sort[np.argmax(x_list[index_y_sort[2:]]) + 2] + + alpha1 = float(box[index_upper_right][1] - box[index_upper_left][1]) / (float(box[index_upper_right][0] - box[index_upper_left][0])) + alpha2 = float(box[index_lower_right][1] - box[index_lower_left][1]) / (float(box[index_lower_right][0] - box[index_lower_left][0])) + + slope_true = (alpha1 + alpha2) / 2.0 + + # slope=0#slope_true/np.pi*180 + + # if abs(slope)>=1: + # slope=0 + + # dst=rotate_image(textline_mask,slope_true) + # dst=dst[:,:,0] + # dst[dst!=0]=1 + image_regions_deskewd = np.zeros(textline_mask_org[:, :].shape) + for ind in np.unique(textline_mask_org[:, :]): + interest_reg = (textline_mask_org[:, :] == ind) * 1 + interest_reg = interest_reg.astype(np.uint8) + deskewed_new = rotate_image(interest_reg, slope_true) + deskewed_new = deskewed_new[:, :] + deskewed_new[deskewed_new != 0] = ind + + image_regions_deskewd = image_regions_deskewd + deskewed_new + return image_regions_deskewd, slope_true + +def get_all_image_patches_coordination(self, image_page): + self.all_box_coord = [] + for jk in range(len(self.boxes)): + _, crop_coor = crop_image_inside_box(self.boxes[jk], image_page) + self.all_box_coord.append(crop_coor) + +def find_num_col_olddd(self, regions_without_seperators, sigma_, multiplier=3.8): + regions_without_seperators_0 = regions_without_seperators[:, :].sum(axis=1) + + meda_n_updown = regions_without_seperators_0[len(regions_without_seperators_0) :: -1] + + first_nonzero = next((i for i, x in enumerate(regions_without_seperators_0) if x), 0) + last_nonzero = next((i for i, x in enumerate(meda_n_updown) if x), 0) + + last_nonzero = len(regions_without_seperators_0) - last_nonzero + + y = regions_without_seperators_0 # [first_nonzero:last_nonzero] + + y_help = np.zeros(len(y) + 20) + + y_help[10 : len(y) + 10] = y + + x = np.array(range(len(y))) + + zneg_rev = -y_help + np.max(y_help) + + zneg = np.zeros(len(zneg_rev) + 20) + + zneg[10 : len(zneg_rev) + 10] = zneg_rev + + z = gaussian_filter1d(y, sigma_) + zneg = gaussian_filter1d(zneg, sigma_) + + peaks_neg, _ = find_peaks(zneg, height=0) + peaks, _ = find_peaks(z, height=0) + + peaks_neg = peaks_neg - 10 - 10 + + last_nonzero = last_nonzero - 0 # 100 + first_nonzero = first_nonzero + 0 # +100 + + peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & (peaks_neg < last_nonzero)] + + peaks = peaks[(peaks > 0.06 * regions_without_seperators.shape[1]) & (peaks < 0.94 * regions_without_seperators.shape[1])] + + interest_pos = z[peaks] + + interest_pos = interest_pos[interest_pos > 10] + + interest_neg = z[peaks_neg] + + if interest_neg[0] < 0.1: + interest_neg = interest_neg[1:] + if interest_neg[len(interest_neg) - 1] < 0.1: + interest_neg = interest_neg[: len(interest_neg) - 1] + + min_peaks_pos = np.min(interest_pos) + min_peaks_neg = 0 # np.min(interest_neg) + + dis_talaei = (min_peaks_pos - min_peaks_neg) / multiplier + grenze = min_peaks_pos - dis_talaei # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 + + interest_neg_fin = interest_neg # [(interest_neg= top) & ((matrix_of_orders[:, 3] < down))] + cxs_in = matrix_of_orders[:, 2][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + + sorted_inside = np.argsort(cxs_in) + + ind_in_int = indexes_in[sorted_inside] + + for j in range(len(ind_in_int)): + final_indexers_sorted.append(int(ind_in_int[j])) + + return final_indexers_sorted, matrix_of_orders + +def remove_headers_and_mains_intersection(seperators_closeup_n, img_revised_tab, boxes): + for ind in range(len(boxes)): + asp = np.zeros((img_revised_tab[:, :, 0].shape[0], seperators_closeup_n[:, :, 0].shape[1])) + asp[int(boxes[ind][2]) : int(boxes[ind][3]), int(boxes[ind][0]) : int(boxes[ind][1])] = img_revised_tab[int(boxes[ind][2]) : int(boxes[ind][3]), int(boxes[ind][0]) : int(boxes[ind][1]), 0] + + head_patch_con = (asp[:, :] == 2) * 1 + main_patch_con = (asp[:, :] == 1) * 1 + # print(head_patch_con) + head_patch_con = head_patch_con.astype(np.uint8) + main_patch_con = main_patch_con.astype(np.uint8) + + head_patch_con = np.repeat(head_patch_con[:, :, np.newaxis], 3, axis=2) + main_patch_con = np.repeat(main_patch_con[:, :, np.newaxis], 3, axis=2) + + imgray = cv2.cvtColor(head_patch_con, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_head_patch_con, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_head_patch_con = return_parent_contours(contours_head_patch_con, hiearchy) + + imgray = cv2.cvtColor(main_patch_con, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_main_patch_con, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_main_patch_con = return_parent_contours(contours_main_patch_con, hiearchy) + + y_patch_head_min, y_patch_head_max, _ = find_features_of_contours(contours_head_patch_con) + y_patch_main_min, y_patch_main_max, _ = find_features_of_contours(contours_main_patch_con) + + for i in range(len(y_patch_head_min)): + for j in range(len(y_patch_main_min)): + if y_patch_head_max[i] > y_patch_main_min[j] and y_patch_head_min[i] < y_patch_main_min[j]: + y_down = y_patch_head_max[i] + y_up = y_patch_main_min[j] + + patch_intersection = np.zeros(asp.shape) + patch_intersection[y_up:y_down, :] = asp[y_up:y_down, :] + + head_patch_con = (patch_intersection[:, :] == 2) * 1 + main_patch_con = (patch_intersection[:, :] == 1) * 1 + head_patch_con = head_patch_con.astype(np.uint8) + main_patch_con = main_patch_con.astype(np.uint8) + + head_patch_con = np.repeat(head_patch_con[:, :, np.newaxis], 3, axis=2) + main_patch_con = np.repeat(main_patch_con[:, :, np.newaxis], 3, axis=2) + + imgray = cv2.cvtColor(head_patch_con, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_head_patch_con, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_head_patch_con = return_parent_contours(contours_head_patch_con, hiearchy) + + imgray = cv2.cvtColor(main_patch_con, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_main_patch_con, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_main_patch_con = return_parent_contours(contours_main_patch_con, hiearchy) + + _, _, areas_head = find_features_of_contours(contours_head_patch_con) + _, _, areas_main = find_features_of_contours(contours_main_patch_con) + + if np.sum(areas_head) > np.sum(areas_main): + img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0][img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0] == 1] = 2 + else: + img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0][img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0] == 2] = 1 + + elif y_patch_head_min[i] < y_patch_main_max[j] and y_patch_head_max[i] > y_patch_main_max[j]: + y_down = y_patch_main_max[j] + y_up = y_patch_head_min[i] + + patch_intersection = np.zeros(asp.shape) + patch_intersection[y_up:y_down, :] = asp[y_up:y_down, :] + + head_patch_con = (patch_intersection[:, :] == 2) * 1 + main_patch_con = (patch_intersection[:, :] == 1) * 1 + head_patch_con = head_patch_con.astype(np.uint8) + main_patch_con = main_patch_con.astype(np.uint8) + + head_patch_con = np.repeat(head_patch_con[:, :, np.newaxis], 3, axis=2) + main_patch_con = np.repeat(main_patch_con[:, :, np.newaxis], 3, axis=2) + + imgray = cv2.cvtColor(head_patch_con, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_head_patch_con, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_head_patch_con = return_parent_contours(contours_head_patch_con, hiearchy) + + imgray = cv2.cvtColor(main_patch_con, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_main_patch_con, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_main_patch_con = return_parent_contours(contours_main_patch_con, hiearchy) + + _, _, areas_head = find_features_of_contours(contours_head_patch_con) + _, _, areas_main = find_features_of_contours(contours_main_patch_con) + + if np.sum(areas_head) > np.sum(areas_main): + img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0][img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0] == 1] = 2 + else: + img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0][img_revised_tab[y_up:y_down, int(boxes[ind][0]) : int(boxes[ind][1]), 0] == 2] = 1 + + # print(np.unique(patch_intersection) ) + ##plt.figure(figsize=(20,20)) + ##plt.imshow(patch_intersection) + ##plt.show() + else: + pass + + return img_revised_tab + +def tear_main_texts_on_the_boundaries_of_boxes(img_revised_tab, boxes): + for i in range(len(boxes)): + img_revised_tab[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][1] - 10) : int(boxes[i][1]), 0][img_revised_tab[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][1] - 10) : int(boxes[i][1]), 0] == 1] = 0 + img_revised_tab[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][1] - 10) : int(boxes[i][1]), 1][img_revised_tab[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][1] - 10) : int(boxes[i][1]), 1] == 1] = 0 + img_revised_tab[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][1] - 10) : int(boxes[i][1]), 2][img_revised_tab[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][1] - 10) : int(boxes[i][1]), 2] == 1] = 0 + return img_revised_tab + +def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back(self, regions_pre_p): + seperators_closeup = ((regions_pre_p[:, :] == 6)) * 1 + + seperators_closeup = seperators_closeup.astype(np.uint8) + kernel = np.ones((5, 5), np.uint8) + + seperators_closeup = cv2.dilate(seperators_closeup, kernel, iterations=1) + seperators_closeup = cv2.erode(seperators_closeup, kernel, iterations=1) + + seperators_closeup = cv2.erode(seperators_closeup, kernel, iterations=1) + seperators_closeup = cv2.dilate(seperators_closeup, kernel, iterations=1) + + if len(seperators_closeup.shape) == 2: + seperators_closeup_n = np.zeros((seperators_closeup.shape[0], seperators_closeup.shape[1], 3)) + seperators_closeup_n[:, :, 0] = seperators_closeup + seperators_closeup_n[:, :, 1] = seperators_closeup + seperators_closeup_n[:, :, 2] = seperators_closeup + else: + seperators_closeup_n = seperators_closeup[:, :, :] + # seperators_closeup=seperators_closeup.astype(np.uint8) + seperators_closeup_n = seperators_closeup_n.astype(np.uint8) + imgray = cv2.cvtColor(seperators_closeup_n, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_lines, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = find_features_of_lines(contours_lines) + + dist_y = np.abs(y_max_main - y_min_main) + + slope_lines_org_hor = slope_lines_org[slope_lines == 0] + args = np.array(range(len(slope_lines))) + len_x = seperators_closeup.shape[1] * 0 + len_y = seperators_closeup.shape[0] * 0.01 + + args_hor = args[slope_lines == 0] + dist_x_hor = dist_x[slope_lines == 0] + dist_y_hor = dist_y[slope_lines == 0] + x_min_main_hor = x_min_main[slope_lines == 0] + x_max_main_hor = x_max_main[slope_lines == 0] + cy_main_hor = cy_main[slope_lines == 0] + y_min_main_hor = y_min_main[slope_lines == 0] + y_max_main_hor = y_max_main[slope_lines == 0] + + args_hor = args_hor[dist_x_hor >= len_x] + x_max_main_hor = x_max_main_hor[dist_x_hor >= len_x] + x_min_main_hor = x_min_main_hor[dist_x_hor >= len_x] + cy_main_hor = cy_main_hor[dist_x_hor >= len_x] + y_min_main_hor = y_min_main_hor[dist_x_hor >= len_x] + y_max_main_hor = y_max_main_hor[dist_x_hor >= len_x] + slope_lines_org_hor = slope_lines_org_hor[dist_x_hor >= len_x] + dist_y_hor = dist_y_hor[dist_x_hor >= len_x] + dist_x_hor = dist_x_hor[dist_x_hor >= len_x] + + args_ver = args[slope_lines == 1] + dist_y_ver = dist_y[slope_lines == 1] + dist_x_ver = dist_x[slope_lines == 1] + x_min_main_ver = x_min_main[slope_lines == 1] + x_max_main_ver = x_max_main[slope_lines == 1] + y_min_main_ver = y_min_main[slope_lines == 1] + y_max_main_ver = y_max_main[slope_lines == 1] + cx_main_ver = cx_main[slope_lines == 1] + + args_ver = args_ver[dist_y_ver >= len_y] + x_max_main_ver = x_max_main_ver[dist_y_ver >= len_y] + x_min_main_ver = x_min_main_ver[dist_y_ver >= len_y] + cx_main_ver = cx_main_ver[dist_y_ver >= len_y] + y_min_main_ver = y_min_main_ver[dist_y_ver >= len_y] + y_max_main_ver = y_max_main_ver[dist_y_ver >= len_y] + dist_x_ver = dist_x_ver[dist_y_ver >= len_y] + dist_y_ver = dist_y_ver[dist_y_ver >= len_y] + + img_p_in_ver = np.zeros(seperators_closeup_n[:, :, 2].shape) + for jv in range(len(args_ver)): + img_p_in_ver = cv2.fillPoly(img_p_in_ver, pts=[contours_lines[args_ver[jv]]], color=(1, 1, 1)) + + img_in_hor = np.zeros(seperators_closeup_n[:, :, 2].shape) + for jv in range(len(args_hor)): + img_p_in_hor = cv2.fillPoly(img_in_hor, pts=[contours_lines[args_hor[jv]]], color=(1, 1, 1)) + + all_args_uniq = contours_in_same_horizon(cy_main_hor) + # print(all_args_uniq,'all_args_uniq') + if len(all_args_uniq) > 0: + if type(all_args_uniq[0]) is list: + contours_new = [] + for dd in range(len(all_args_uniq)): + merged_all = None + some_args = args_hor[all_args_uniq[dd]] + some_cy = cy_main_hor[all_args_uniq[dd]] + some_x_min = x_min_main_hor[all_args_uniq[dd]] + some_x_max = x_max_main_hor[all_args_uniq[dd]] + + img_in = np.zeros(seperators_closeup_n[:, :, 2].shape) + for jv in range(len(some_args)): + + img_p_in = cv2.fillPoly(img_p_in_hor, pts=[contours_lines[some_args[jv]]], color=(1, 1, 1)) + img_p_in[int(np.mean(some_cy)) - 5 : int(np.mean(some_cy)) + 5, int(np.min(some_x_min)) : int(np.max(some_x_max))] = 1 + + else: + img_p_in = seperators_closeup + else: + img_p_in = seperators_closeup + + sep_ver_hor = img_p_in + img_p_in_ver + sep_ver_hor_cross = (sep_ver_hor == 2) * 1 + + sep_ver_hor_cross = np.repeat(sep_ver_hor_cross[:, :, np.newaxis], 3, axis=2) + sep_ver_hor_cross = sep_ver_hor_cross.astype(np.uint8) + imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_cross, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + cx_cross, cy_cross, _, _, _, _, _ = find_new_features_of_contoures(contours_cross) + + for ii in range(len(cx_cross)): + sep_ver_hor[int(cy_cross[ii]) - 15 : int(cy_cross[ii]) + 15, int(cx_cross[ii]) + 5 : int(cx_cross[ii]) + 40] = 0 + sep_ver_hor[int(cy_cross[ii]) - 15 : int(cy_cross[ii]) + 15, int(cx_cross[ii]) - 40 : int(cx_cross[ii]) - 4] = 0 + + img_p_in[:, :] = sep_ver_hor[:, :] + + if len(img_p_in.shape) == 2: + seperators_closeup_n = np.zeros((img_p_in.shape[0], img_p_in.shape[1], 3)) + seperators_closeup_n[:, :, 0] = img_p_in + seperators_closeup_n[:, :, 1] = img_p_in + seperators_closeup_n[:, :, 2] = img_p_in + else: + seperators_closeup_n = img_p_in[:, :, :] + # seperators_closeup=seperators_closeup.astype(np.uint8) + seperators_closeup_n = seperators_closeup_n.astype(np.uint8) + imgray = cv2.cvtColor(seperators_closeup_n, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_lines, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = find_features_of_lines(contours_lines) + + dist_y = np.abs(y_max_main - y_min_main) + + slope_lines_org_hor = slope_lines_org[slope_lines == 0] + args = np.array(range(len(slope_lines))) + len_x = seperators_closeup.shape[1] * 0.04 + len_y = seperators_closeup.shape[0] * 0.08 + + args_hor = args[slope_lines == 0] + dist_x_hor = dist_x[slope_lines == 0] + dist_y_hor = dist_y[slope_lines == 0] + x_min_main_hor = x_min_main[slope_lines == 0] + x_max_main_hor = x_max_main[slope_lines == 0] + cy_main_hor = cy_main[slope_lines == 0] + y_min_main_hor = y_min_main[slope_lines == 0] + y_max_main_hor = y_max_main[slope_lines == 0] + + args_hor = args_hor[dist_x_hor >= len_x] + x_max_main_hor = x_max_main_hor[dist_x_hor >= len_x] + x_min_main_hor = x_min_main_hor[dist_x_hor >= len_x] + cy_main_hor = cy_main_hor[dist_x_hor >= len_x] + y_min_main_hor = y_min_main_hor[dist_x_hor >= len_x] + y_max_main_hor = y_max_main_hor[dist_x_hor >= len_x] + slope_lines_org_hor = slope_lines_org_hor[dist_x_hor >= len_x] + dist_y_hor = dist_y_hor[dist_x_hor >= len_x] + dist_x_hor = dist_x_hor[dist_x_hor >= len_x] + + args_ver = args[slope_lines == 1] + dist_y_ver = dist_y[slope_lines == 1] + dist_x_ver = dist_x[slope_lines == 1] + x_min_main_ver = x_min_main[slope_lines == 1] + x_max_main_ver = x_max_main[slope_lines == 1] + y_min_main_ver = y_min_main[slope_lines == 1] + y_max_main_ver = y_max_main[slope_lines == 1] + cx_main_ver = cx_main[slope_lines == 1] + + args_ver = args_ver[dist_y_ver >= len_y] + x_max_main_ver = x_max_main_ver[dist_y_ver >= len_y] + x_min_main_ver = x_min_main_ver[dist_y_ver >= len_y] + cx_main_ver = cx_main_ver[dist_y_ver >= len_y] + y_min_main_ver = y_min_main_ver[dist_y_ver >= len_y] + y_max_main_ver = y_max_main_ver[dist_y_ver >= len_y] + dist_x_ver = dist_x_ver[dist_y_ver >= len_y] + dist_y_ver = dist_y_ver[dist_y_ver >= len_y] + + matrix_of_lines_ch = np.zeros((len(cy_main_hor) + len(cx_main_ver), 10)) + + matrix_of_lines_ch[: len(cy_main_hor), 0] = args_hor + matrix_of_lines_ch[len(cy_main_hor) :, 0] = args_ver + + matrix_of_lines_ch[len(cy_main_hor) :, 1] = cx_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 2] = x_min_main_hor + matrix_of_lines_ch[len(cy_main_hor) :, 2] = x_min_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 3] = x_max_main_hor + matrix_of_lines_ch[len(cy_main_hor) :, 3] = x_max_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 4] = dist_x_hor + matrix_of_lines_ch[len(cy_main_hor) :, 4] = dist_x_ver + + matrix_of_lines_ch[: len(cy_main_hor), 5] = cy_main_hor + + matrix_of_lines_ch[: len(cy_main_hor), 6] = y_min_main_hor + matrix_of_lines_ch[len(cy_main_hor) :, 6] = y_min_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 7] = y_max_main_hor + matrix_of_lines_ch[len(cy_main_hor) :, 7] = y_max_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 8] = dist_y_hor + matrix_of_lines_ch[len(cy_main_hor) :, 8] = dist_y_ver + + matrix_of_lines_ch[len(cy_main_hor) :, 9] = 1 + + return matrix_of_lines_ch, seperators_closeup_n + +def image_change_background_pixels_to_zero(self, image_page): + image_back_zero = np.zeros((image_page.shape[0], image_page.shape[1])) + image_back_zero[:, :] = image_page[:, :, 0] + image_back_zero[:, :][image_back_zero[:, :] == 0] = -255 + image_back_zero[:, :][image_back_zero[:, :] == 255] = 0 + image_back_zero[:, :][image_back_zero[:, :] == -255] = 255 + return image_back_zero + +def return_boxes_of_images_by_order_of_reading_without_seperator(spliter_y_new, image_p_rev, regions_without_seperators, matrix_of_lines_ch, seperators_closeup_n): + + boxes = [] + + # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ + # holes in the text and also finding spliter which covers more than one columns. + for i in range(len(spliter_y_new) - 1): + # print(spliter_y_new[i],spliter_y_new[i+1]) + matrix_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 6] > spliter_y_new[i]) & (matrix_of_lines_ch[:, 7] < spliter_y_new[i + 1])] + # print(len( matrix_new[:,9][matrix_new[:,9]==1] )) + + # print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') + + # check to see is there any vertical seperator to find holes. + if np.abs(spliter_y_new[i + 1] - spliter_y_new[i]) > 1.0 / 3.0 * regions_without_seperators.shape[0]: # len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): + + # org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) + # org_img_dichte=org_img_dichte-np.min(org_img_dichte) + ##plt.figure(figsize=(20,20)) + ##plt.plot(org_img_dichte) + ##plt.show() + ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) + + num_col, peaks_neg_fin = find_num_col_only_image(image_p_rev[int(spliter_y_new[i]) : int(spliter_y_new[i + 1]), :], multiplier=2.4) + + # num_col, peaks_neg_fin=find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) + x_min_hor_some = matrix_new[:, 2][(matrix_new[:, 9] == 0)] + x_max_hor_some = matrix_new[:, 3][(matrix_new[:, 9] == 0)] + cy_hor_some = matrix_new[:, 5][(matrix_new[:, 9] == 0)] + arg_org_hor_some = matrix_new[:, 0][(matrix_new[:, 9] == 0)] + + peaks_neg_tot = return_points_with_boundies(peaks_neg_fin, 0, seperators_closeup_n[:, :, 0].shape[1]) + + start_index_of_hor, newest_peaks, arg_min_hor_sort, lines_length_dels, lines_indexes_deleted = return_hor_spliter_by_index_for_without_verticals(peaks_neg_tot, x_min_hor_some, x_max_hor_some) + + arg_org_hor_some_sort = arg_org_hor_some[arg_min_hor_sort] + + start_index_of_hor_with_subset = [start_index_of_hor[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] # start_index_of_hor[lines_length_dels>0] + arg_min_hor_sort_with_subset = [arg_min_hor_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + lines_indexes_deleted_with_subset = [lines_indexes_deleted[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + lines_length_dels_with_subset = [lines_length_dels[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + + arg_org_hor_some_sort_subset = [arg_org_hor_some_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + + # arg_min_hor_sort_with_subset=arg_min_hor_sort[lines_length_dels>0] + # lines_indexes_deleted_with_subset=lines_indexes_deleted[lines_length_dels>0] + # lines_length_dels_with_subset=lines_length_dels[lines_length_dels>0] + + # print(len(arg_min_hor_sort),len(arg_org_hor_some_sort),'vizzzzzz') + + vahid_subset = np.zeros((len(start_index_of_hor_with_subset), len(start_index_of_hor_with_subset))) - 1 + for kkk1 in range(len(start_index_of_hor_with_subset)): + + # print(lines_indexes_deleted,'hiii') + index_del_sub = np.unique(lines_indexes_deleted_with_subset[kkk1]) + + for kkk2 in range(len(start_index_of_hor_with_subset)): + + if set(lines_indexes_deleted_with_subset[kkk2][0]) < set(lines_indexes_deleted_with_subset[kkk1][0]): + vahid_subset[kkk1, kkk2] = kkk1 + else: + pass + # print(set(lines_indexes_deleted[kkk2][0]), set(lines_indexes_deleted[kkk1][0])) + + # check the len of matrix if it has no length means that there is no spliter at all + + if len(vahid_subset > 0): + # print('hihoo') + + # find parenets args + line_int = np.zeros(vahid_subset.shape[0]) + + childs_id = [] + arg_child = [] + for li in range(vahid_subset.shape[0]): + if np.all(vahid_subset[:, li] == -1): + line_int[li] = -1 + else: + line_int[li] = 1 + + # childs_args_in=[ idd for idd in range(vahid_subset.shape[0]) if vahid_subset[idd,li]!=-1] + # helpi=[] + # for nad in range(len(childs_args_in)): + # helpi.append(arg_min_hor_sort_with_subset[childs_args_in[nad]]) + + arg_child.append(arg_min_hor_sort_with_subset[li]) + + arg_parent = [arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] == -1] + start_index_of_hor_parent = [start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] == -1] + # arg_parent=[lines_indexes_deleted_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] + # arg_parent=[lines_length_dels_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] + + # arg_child=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] + start_index_of_hor_child = [start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] != -1] + + cy_hor_some_sort = cy_hor_some[arg_parent] + + newest_y_spliter_tot = [] + + for tj in range(len(newest_peaks) - 1): + newest_y_spliter = [] + newest_y_spliter.append(spliter_y_new[i]) + if tj in np.unique(start_index_of_hor_parent): + cy_help = np.array(cy_hor_some_sort)[np.array(start_index_of_hor_parent) == tj] + cy_help_sort = np.sort(cy_help) + + # print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') + for mj in range(len(cy_help_sort)): + newest_y_spliter.append(cy_help_sort[mj]) + newest_y_spliter.append(spliter_y_new[i + 1]) + + newest_y_spliter_tot.append(newest_y_spliter) + + else: + line_int = [] + newest_y_spliter_tot = [] + + for tj in range(len(newest_peaks) - 1): + newest_y_spliter = [] + newest_y_spliter.append(spliter_y_new[i]) + + newest_y_spliter.append(spliter_y_new[i + 1]) + + newest_y_spliter_tot.append(newest_y_spliter) + + # if line_int is all -1 means that big spliters have no child and we can easily go through + if np.all(np.array(line_int) == -1): + for j in range(len(newest_peaks) - 1): + newest_y_spliter = newest_y_spliter_tot[j] + + for n in range(len(newest_y_spliter) - 1): + # print(j,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'maaaa') + ##plt.imshow(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]]) + ##plt.show() + + # print(matrix_new[:,0][ (matrix_new[:,9]==1 )]) + for jvt in matrix_new[:, 0][(matrix_new[:, 9] == 1) & (matrix_new[:, 6] > newest_y_spliter[n]) & (matrix_new[:, 7] < newest_y_spliter[n + 1]) & ((matrix_new[:, 1]) < newest_peaks[j + 1]) & ((matrix_new[:, 1]) > newest_peaks[j])]: + pass + + ###plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) + # print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): + # num_col_sub, peaks_neg_fin_sub=find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) + num_col_sub, peaks_neg_fin_sub = find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=2.4) + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + for j in range(len(newest_peaks) - 1): + newest_y_spliter = newest_y_spliter_tot[j] + + if j in start_index_of_hor_parent: + + x_min_ch = x_min_hor_some[arg_child] + x_max_ch = x_max_hor_some[arg_child] + cy_hor_some_sort_child = cy_hor_some[arg_child] + cy_hor_some_sort_child = np.sort(cy_hor_some_sort_child) + + for n in range(len(newest_y_spliter) - 1): + + cy_child_in = cy_hor_some_sort_child[(cy_hor_some_sort_child > newest_y_spliter[n]) & (cy_hor_some_sort_child < newest_y_spliter[n + 1])] + + if len(cy_child_in) > 0: + ###num_col_ch, peaks_neg_ch=find_num_col( regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) + + num_col_ch, peaks_neg_ch = find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=2.3) + + peaks_neg_ch = peaks_neg_ch[:] + newest_peaks[j] + + peaks_neg_ch_tot = return_points_with_boundies(peaks_neg_ch, newest_peaks[j], newest_peaks[j + 1]) + + ss_in_ch, nst_p_ch, arg_n_ch, lines_l_del_ch, lines_in_del_ch = return_hor_spliter_by_index_for_without_verticals(peaks_neg_ch_tot, x_min_ch, x_max_ch) + + newest_y_spliter_ch_tot = [] + + for tjj in range(len(nst_p_ch) - 1): + newest_y_spliter_new = [] + newest_y_spliter_new.append(newest_y_spliter[n]) + if tjj in np.unique(ss_in_ch): + + # print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') + for mjj in range(len(cy_child_in)): + newest_y_spliter_new.append(cy_child_in[mjj]) + newest_y_spliter_new.append(newest_y_spliter[n + 1]) + + newest_y_spliter_ch_tot.append(newest_y_spliter_new) + + for jn in range(len(nst_p_ch) - 1): + newest_y_spliter_h = newest_y_spliter_ch_tot[jn] + + for nd in range(len(newest_y_spliter_h) - 1): + + matrix_new_new2 = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter_h[nd]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter_h[nd + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < nst_p_ch[jn + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > nst_p_ch[jn])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new2[:,9][matrix_new_new2[:,9]==1] )>0 and np.max(matrix_new_new2[:,8][matrix_new_new2[:,9]==1])>=0.2*(np.abs(newest_y_spliter_h[nd+1]-newest_y_spliter_h[nd] )): + # num_col_sub_ch, peaks_neg_fin_sub_ch=find_num_col(regions_without_seperators[int(newest_y_spliter_h[nd]):int(newest_y_spliter_h[nd+1]),nst_p_ch[jn]:nst_p_ch[jn+1]],multiplier=2.3) + + num_col_sub_ch, peaks_neg_fin_sub_ch = find_num_col_only_image(image_p_rev[int(newest_y_spliter_h[nd]) : int(newest_y_spliter_h[nd + 1]), nst_p_ch[jn] : nst_p_ch[jn + 1]], multiplier=2.3) + # print(peaks_neg_fin_sub_ch,'gada kutullllllll') + else: + peaks_neg_fin_sub_ch = [] + + peaks_sub_ch = [] + peaks_sub_ch.append(nst_p_ch[jn]) + + for kjj in range(len(peaks_neg_fin_sub_ch)): + peaks_sub_ch.append(peaks_neg_fin_sub_ch[kjj] + nst_p_ch[jn]) + + peaks_sub_ch.append(nst_p_ch[jn + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for khh in range(len(peaks_sub_ch) - 1): + boxes.append([peaks_sub_ch[khh], peaks_sub_ch[khh + 1], newest_y_spliter_h[nd], newest_y_spliter_h[nd + 1]]) + + else: + + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): + ###num_col_sub, peaks_neg_fin_sub=find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=2.3) + num_col_sub, peaks_neg_fin_sub = find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=2.3) + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + for n in range(len(newest_y_spliter) - 1): + + for jvt in matrix_new[:, 0][(matrix_new[:, 9] == 1) & (matrix_new[:, 6] > newest_y_spliter[n]) & (matrix_new[:, 7] < newest_y_spliter[n + 1]) & ((matrix_new[:, 1]) < newest_peaks[j + 1]) & ((matrix_new[:, 1]) > newest_peaks[j])]: + pass + + # plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) + # print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): + ###num_col_sub, peaks_neg_fin_sub=find_num_col(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]],multiplier=5.0) + num_col_sub, peaks_neg_fin_sub = find_num_col_only_image(image_p_rev[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=2.3) + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + boxes.append([0, seperators_closeup_n[:, :, 0].shape[1], spliter_y_new[i], spliter_y_new[i + 1]]) + return boxes + +def return_region_segmentation_after_implementing_not_head_maintext_parallel(image_regions_eraly_p, boxes): + image_revised = np.zeros((image_regions_eraly_p.shape[0], image_regions_eraly_p.shape[1])) + for i in range(len(boxes)): + + image_box = image_regions_eraly_p[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][0]) : int(boxes[i][1])] + image_box = np.array(image_box) + # plt.imshow(image_box) + # plt.show() + + # print(int(boxes[i][2]),int(boxes[i][3]),int(boxes[i][0]),int(boxes[i][1]),'addaa') + image_box = implent_law_head_main_not_parallel(image_box) + image_box = implent_law_head_main_not_parallel(image_box) + image_box = implent_law_head_main_not_parallel(image_box) + + image_revised[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][0]) : int(boxes[i][1])] = image_box[:, :] + return image_revised + +def return_boxes_of_images_by_order_of_reading_2cols(spliter_y_new, regions_without_seperators, matrix_of_lines_ch, seperators_closeup_n): + boxes = [] + + # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ + # holes in the text and also finding spliter which covers more than one columns. + for i in range(len(spliter_y_new) - 1): + # print(spliter_y_new[i],spliter_y_new[i+1]) + matrix_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 6] > spliter_y_new[i]) & (matrix_of_lines_ch[:, 7] < spliter_y_new[i + 1])] + # print(len( matrix_new[:,9][matrix_new[:,9]==1] )) + + # print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') + + # check to see is there any vertical seperator to find holes. + if 1 > 0: # len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): + # print(int(spliter_y_new[i]),int(spliter_y_new[i+1]),'burayaaaa galimiirrrrrrrrrrrrrrrrrrrrrrrrrrr') + # org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) + # org_img_dichte=org_img_dichte-np.min(org_img_dichte) + ##plt.figure(figsize=(20,20)) + ##plt.plot(org_img_dichte) + ##plt.show() + ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) + + try: + num_col, peaks_neg_fin = find_num_col(regions_without_seperators[int(spliter_y_new[i]) : int(spliter_y_new[i + 1]), :], multiplier=7.0) + + except: + peaks_neg_fin = [] + num_col = 0 + + peaks_neg_tot = return_points_with_boundies(peaks_neg_fin, 0, seperators_closeup_n[:, :, 0].shape[1]) + + for kh in range(len(peaks_neg_tot) - 1): + boxes.append([peaks_neg_tot[kh], peaks_neg_tot[kh + 1], spliter_y_new[i], spliter_y_new[i + 1]]) + + else: + boxes.append([0, seperators_closeup_n[:, :, 0].shape[1], spliter_y_new[i], spliter_y_new[i + 1]]) + + return boxes + +def return_boxes_of_images_by_order_of_reading(spliter_y_new, regions_without_seperators, matrix_of_lines_ch, seperators_closeup_n): + boxes = [] + + # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ + # holes in the text and also finding spliter which covers more than one columns. + for i in range(len(spliter_y_new) - 1): + # print(spliter_y_new[i],spliter_y_new[i+1]) + matrix_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 6] > spliter_y_new[i]) & (matrix_of_lines_ch[:, 7] < spliter_y_new[i + 1])] + # print(len( matrix_new[:,9][matrix_new[:,9]==1] )) + + # print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') + + # check to see is there any vertical seperator to find holes. + if len(matrix_new[:, 9][matrix_new[:, 9] == 1]) > 0 and np.max(matrix_new[:, 8][matrix_new[:, 9] == 1]) >= 0.1 * (np.abs(spliter_y_new[i + 1] - spliter_y_new[i])): + + # org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) + # org_img_dichte=org_img_dichte-np.min(org_img_dichte) + ##plt.figure(figsize=(20,20)) + ##plt.plot(org_img_dichte) + ##plt.show() + ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) + + num_col, peaks_neg_fin = find_num_col(regions_without_seperators[int(spliter_y_new[i]) : int(spliter_y_new[i + 1]), :], multiplier=7.0) + + # num_col, peaks_neg_fin=find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) + x_min_hor_some = matrix_new[:, 2][(matrix_new[:, 9] == 0)] + x_max_hor_some = matrix_new[:, 3][(matrix_new[:, 9] == 0)] + cy_hor_some = matrix_new[:, 5][(matrix_new[:, 9] == 0)] + arg_org_hor_some = matrix_new[:, 0][(matrix_new[:, 9] == 0)] + + peaks_neg_tot = return_points_with_boundies(peaks_neg_fin, 0, seperators_closeup_n[:, :, 0].shape[1]) + + start_index_of_hor, newest_peaks, arg_min_hor_sort, lines_length_dels, lines_indexes_deleted = return_hor_spliter_by_index(peaks_neg_tot, x_min_hor_some, x_max_hor_some) + + arg_org_hor_some_sort = arg_org_hor_some[arg_min_hor_sort] + + start_index_of_hor_with_subset = [start_index_of_hor[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] # start_index_of_hor[lines_length_dels>0] + arg_min_hor_sort_with_subset = [arg_min_hor_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + lines_indexes_deleted_with_subset = [lines_indexes_deleted[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + lines_length_dels_with_subset = [lines_length_dels[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + + arg_org_hor_some_sort_subset = [arg_org_hor_some_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + + # arg_min_hor_sort_with_subset=arg_min_hor_sort[lines_length_dels>0] + # lines_indexes_deleted_with_subset=lines_indexes_deleted[lines_length_dels>0] + # lines_length_dels_with_subset=lines_length_dels[lines_length_dels>0] + + vahid_subset = np.zeros((len(start_index_of_hor_with_subset), len(start_index_of_hor_with_subset))) - 1 + for kkk1 in range(len(start_index_of_hor_with_subset)): + + index_del_sub = np.unique(lines_indexes_deleted_with_subset[kkk1]) + + for kkk2 in range(len(start_index_of_hor_with_subset)): + + if set(lines_indexes_deleted_with_subset[kkk2][0]) < set(lines_indexes_deleted_with_subset[kkk1][0]): + vahid_subset[kkk1, kkk2] = kkk1 + else: + pass + # print(set(lines_indexes_deleted[kkk2][0]), set(lines_indexes_deleted[kkk1][0])) + + # print(vahid_subset,'zartt222') + + # check the len of matrix if it has no length means that there is no spliter at all + + if len(vahid_subset > 0): + # print('hihoo') + + # find parenets args + line_int = np.zeros(vahid_subset.shape[0]) + + childs_id = [] + arg_child = [] + for li in range(vahid_subset.shape[0]): + # print(vahid_subset[:,li]) + if np.all(vahid_subset[:, li] == -1): + line_int[li] = -1 + else: + line_int[li] = 1 + + # childs_args_in=[ idd for idd in range(vahid_subset.shape[0]) if vahid_subset[idd,li]!=-1] + # helpi=[] + # for nad in range(len(childs_args_in)): + # helpi.append(arg_min_hor_sort_with_subset[childs_args_in[nad]]) + + arg_child.append(arg_min_hor_sort_with_subset[li]) + + # line_int=vahid_subset[0,:] + + # print(arg_child,line_int[0],'zartt33333') + arg_parent = [arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] == -1] + start_index_of_hor_parent = [start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] == -1] + # arg_parent=[lines_indexes_deleted_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] + # arg_parent=[lines_length_dels_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] + + # arg_child=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] + start_index_of_hor_child = [start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] != -1] + + cy_hor_some_sort = cy_hor_some[arg_parent] + + # print(start_index_of_hor, lines_length_dels ,lines_indexes_deleted,'zartt') + + # args_indexes=np.array(range(len(start_index_of_hor) )) + + newest_y_spliter_tot = [] + + for tj in range(len(newest_peaks) - 1): + newest_y_spliter = [] + newest_y_spliter.append(spliter_y_new[i]) + if tj in np.unique(start_index_of_hor_parent): + ##print(cy_hor_some_sort) + cy_help = np.array(cy_hor_some_sort)[np.array(start_index_of_hor_parent) == tj] + cy_help_sort = np.sort(cy_help) + + # print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') + for mj in range(len(cy_help_sort)): + newest_y_spliter.append(cy_help_sort[mj]) + newest_y_spliter.append(spliter_y_new[i + 1]) + + newest_y_spliter_tot.append(newest_y_spliter) + + else: + line_int = [] + newest_y_spliter_tot = [] + + for tj in range(len(newest_peaks) - 1): + newest_y_spliter = [] + newest_y_spliter.append(spliter_y_new[i]) + + newest_y_spliter.append(spliter_y_new[i + 1]) + + newest_y_spliter_tot.append(newest_y_spliter) + + # if line_int is all -1 means that big spliters have no child and we can easily go through + if np.all(np.array(line_int) == -1): + for j in range(len(newest_peaks) - 1): + newest_y_spliter = newest_y_spliter_tot[j] + + for n in range(len(newest_y_spliter) - 1): + # print(j,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'maaaa') + ##plt.imshow(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]]) + ##plt.show() + + # print(matrix_new[:,0][ (matrix_new[:,9]==1 )]) + for jvt in matrix_new[:, 0][(matrix_new[:, 9] == 1) & (matrix_new[:, 6] > newest_y_spliter[n]) & (matrix_new[:, 7] < newest_y_spliter[n + 1]) & ((matrix_new[:, 1]) < newest_peaks[j + 1]) & ((matrix_new[:, 1]) > newest_peaks[j])]: + pass + + ###plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) + # print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if len(matrix_new_new[:, 9][matrix_new_new[:, 9] == 1]) > 0 and np.max(matrix_new_new[:, 8][matrix_new_new[:, 9] == 1]) >= 0.2 * (np.abs(newest_y_spliter[n + 1] - newest_y_spliter[n])): + num_col_sub, peaks_neg_fin_sub = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=5.0) + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + for j in range(len(newest_peaks) - 1): + newest_y_spliter = newest_y_spliter_tot[j] + + if j in start_index_of_hor_parent: + + x_min_ch = x_min_hor_some[arg_child] + x_max_ch = x_max_hor_some[arg_child] + cy_hor_some_sort_child = cy_hor_some[arg_child] + cy_hor_some_sort_child = np.sort(cy_hor_some_sort_child) + + # print(cy_hor_some_sort_child,'ychilds') + + for n in range(len(newest_y_spliter) - 1): + + cy_child_in = cy_hor_some_sort_child[(cy_hor_some_sort_child > newest_y_spliter[n]) & (cy_hor_some_sort_child < newest_y_spliter[n + 1])] + + if len(cy_child_in) > 0: + num_col_ch, peaks_neg_ch = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=5.0) + # print(peaks_neg_ch,'mizzzz') + # peaks_neg_ch=[] + # for djh in range(len(peaks_neg_ch)): + # peaks_neg_ch.append( peaks_neg_ch[djh]+newest_peaks[j] ) + + peaks_neg_ch_tot = return_points_with_boundies(peaks_neg_ch, newest_peaks[j], newest_peaks[j + 1]) + + ss_in_ch, nst_p_ch, arg_n_ch, lines_l_del_ch, lines_in_del_ch = return_hor_spliter_by_index(peaks_neg_ch_tot, x_min_ch, x_max_ch) + + newest_y_spliter_ch_tot = [] + + for tjj in range(len(nst_p_ch) - 1): + newest_y_spliter_new = [] + newest_y_spliter_new.append(newest_y_spliter[n]) + if tjj in np.unique(ss_in_ch): + + # print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') + for mjj in range(len(cy_child_in)): + newest_y_spliter_new.append(cy_child_in[mjj]) + newest_y_spliter_new.append(newest_y_spliter[n + 1]) + + newest_y_spliter_ch_tot.append(newest_y_spliter_new) + + for jn in range(len(nst_p_ch) - 1): + newest_y_spliter_h = newest_y_spliter_ch_tot[jn] + + for nd in range(len(newest_y_spliter_h) - 1): + + matrix_new_new2 = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter_h[nd]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter_h[nd + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < nst_p_ch[jn + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > nst_p_ch[jn])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if len(matrix_new_new2[:, 9][matrix_new_new2[:, 9] == 1]) > 0 and np.max(matrix_new_new2[:, 8][matrix_new_new2[:, 9] == 1]) >= 0.2 * (np.abs(newest_y_spliter_h[nd + 1] - newest_y_spliter_h[nd])): + num_col_sub_ch, peaks_neg_fin_sub_ch = find_num_col(regions_without_seperators[int(newest_y_spliter_h[nd]) : int(newest_y_spliter_h[nd + 1]), nst_p_ch[jn] : nst_p_ch[jn + 1]], multiplier=5.0) + + else: + peaks_neg_fin_sub_ch = [] + + peaks_sub_ch = [] + peaks_sub_ch.append(nst_p_ch[jn]) + + for kjj in range(len(peaks_neg_fin_sub_ch)): + peaks_sub_ch.append(peaks_neg_fin_sub_ch[kjj] + nst_p_ch[jn]) + + peaks_sub_ch.append(nst_p_ch[jn + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for khh in range(len(peaks_sub_ch) - 1): + boxes.append([peaks_sub_ch[khh], peaks_sub_ch[khh + 1], newest_y_spliter_h[nd], newest_y_spliter_h[nd + 1]]) + + else: + + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if len(matrix_new_new[:, 9][matrix_new_new[:, 9] == 1]) > 0 and np.max(matrix_new_new[:, 8][matrix_new_new[:, 9] == 1]) >= 0.2 * (np.abs(newest_y_spliter[n + 1] - newest_y_spliter[n])): + num_col_sub, peaks_neg_fin_sub = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=5.0) + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + for n in range(len(newest_y_spliter) - 1): + + # plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) + # print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if len(matrix_new_new[:, 9][matrix_new_new[:, 9] == 1]) > 0 and np.max(matrix_new_new[:, 8][matrix_new_new[:, 9] == 1]) >= 0.2 * (np.abs(newest_y_spliter[n + 1] - newest_y_spliter[n])): + num_col_sub, peaks_neg_fin_sub = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=5.0) + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + boxes.append([0, seperators_closeup_n[:, :, 0].shape[1], spliter_y_new[i], spliter_y_new[i + 1]]) + + return boxes + +def return_boxes_of_images_by_order_of_reading_without_seperators_2cols(spliter_y_new, image_p_rev, regions_without_seperators, matrix_of_lines_ch, seperators_closeup_n): + + boxes = [] + + # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ + # holes in the text and also finding spliter which covers more than one columns. + for i in range(len(spliter_y_new) - 1): + # print(spliter_y_new[i],spliter_y_new[i+1]) + matrix_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 6] > spliter_y_new[i]) & (matrix_of_lines_ch[:, 7] < spliter_y_new[i + 1])] + # print(len( matrix_new[:,9][matrix_new[:,9]==1] )) + + # print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') + + # check to see is there any vertical seperator to find holes. + if np.abs(spliter_y_new[i + 1] - spliter_y_new[i]) > 1.0 / 3.0 * regions_without_seperators.shape[0]: # len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): + + # org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) + # org_img_dichte=org_img_dichte-np.min(org_img_dichte) + ##plt.figure(figsize=(20,20)) + ##plt.plot(org_img_dichte) + ##plt.show() + ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) + + try: + num_col, peaks_neg_fin = find_num_col_only_image(image_p_rev[int(spliter_y_new[i]) : int(spliter_y_new[i + 1]), :], multiplier=2.4) + except: + peaks_neg_fin = [] + num_col = 0 + + peaks_neg_tot = return_points_with_boundies(peaks_neg_fin, 0, seperators_closeup_n[:, :, 0].shape[1]) + + for kh in range(len(peaks_neg_tot) - 1): + boxes.append([peaks_neg_tot[kh], peaks_neg_tot[kh + 1], spliter_y_new[i], spliter_y_new[i + 1]]) + else: + boxes.append([0, seperators_closeup_n[:, :, 0].shape[1], spliter_y_new[i], spliter_y_new[i + 1]]) + + return boxes + +def add_tables_heuristic_to_layout(image_regions_eraly_p, boxes, slope_mean_hor, spliter_y, peaks_neg_tot, image_revised): + + image_revised_1 = delete_seperator_around(spliter_y, peaks_neg_tot, image_revised) + img_comm_e = np.zeros(image_revised_1.shape) + img_comm = np.repeat(img_comm_e[:, :, np.newaxis], 3, axis=2) + + for indiv in np.unique(image_revised_1): + + # print(indiv,'indd') + image_col = (image_revised_1 == indiv) * 255 + img_comm_in = np.repeat(image_col[:, :, np.newaxis], 3, axis=2) + img_comm_in = img_comm_in.astype(np.uint8) + + imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY) + + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours, hirarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area=1, min_area=0.0001) + + img_comm = cv2.fillPoly(img_comm, pts=main_contours, color=(indiv, indiv, indiv)) + ###img_comm_in=cv2.fillPoly(img_comm, pts =interior_contours, color=(0,0,0)) + + # img_comm=np.repeat(img_comm[:, :, np.newaxis], 3, axis=2) + img_comm = img_comm.astype(np.uint8) + + if not isNaN(slope_mean_hor): + image_revised_last = np.zeros((image_regions_eraly_p.shape[0], image_regions_eraly_p.shape[1], 3)) + for i in range(len(boxes)): + + image_box = img_comm[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][0]) : int(boxes[i][1]), :] + + image_box_tabels_1 = (image_box[:, :, 0] == 7) * 1 + + contours_tab, _ = return_contours_of_image(image_box_tabels_1) + + contours_tab = filter_contours_area_of_image_tables(image_box_tabels_1, contours_tab, _, 1, 0.001) + + image_box_tabels_1 = (image_box[:, :, 0] == 6) * 1 + + image_box_tabels_and_m_text = ((image_box[:, :, 0] == 7) | (image_box[:, :, 0] == 1)) * 1 + image_box_tabels_and_m_text = image_box_tabels_and_m_text.astype(np.uint8) + + image_box_tabels_1 = image_box_tabels_1.astype(np.uint8) + image_box_tabels_1 = cv2.dilate(image_box_tabels_1, self.kernel, iterations=5) + + contours_table_m_text, _ = return_contours_of_image(image_box_tabels_and_m_text) + + image_box_tabels = np.repeat(image_box_tabels_1[:, :, np.newaxis], 3, axis=2) + + image_box_tabels = image_box_tabels.astype(np.uint8) + imgray = cv2.cvtColor(image_box_tabels, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_line, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + y_min_main_line, y_max_main_line, _ = find_features_of_contours(contours_line) + # _,_,y_min_main_line ,y_max_main_line,x_min_main_line,x_max_main_line=find_new_features_of_contoures(contours_line) + y_min_main_tab, y_max_main_tab, _ = find_features_of_contours(contours_tab) + + cx_tab_m_text, cy_tab_m_text, x_min_tab_m_text, x_max_tab_m_text, y_min_tab_m_text, y_max_tab_m_text = find_new_features_of_contoures(contours_table_m_text) + cx_tabl, cy_tabl, x_min_tabl, x_max_tabl, y_min_tabl, y_max_tabl, _ = find_new_features_of_contoures(contours_tab) + + if len(y_min_main_tab) > 0: + y_down_tabs = [] + y_up_tabs = [] + + for i_t in range(len(y_min_main_tab)): + y_down_tab = [] + y_up_tab = [] + for i_l in range(len(y_min_main_line)): + if y_min_main_tab[i_t] > y_min_main_line[i_l] and y_max_main_tab[i_t] > y_min_main_line[i_l] and y_min_main_tab[i_t] > y_max_main_line[i_l] and y_max_main_tab[i_t] > y_min_main_line[i_l]: + pass + elif y_min_main_tab[i_t] < y_max_main_line[i_l] and y_max_main_tab[i_t] < y_max_main_line[i_l] and y_max_main_tab[i_t] < y_min_main_line[i_l] and y_min_main_tab[i_t] < y_min_main_line[i_l]: + pass + elif np.abs(y_max_main_line[i_l] - y_min_main_line[i_l]) < 100: + pass + + else: + y_up_tab.append(np.min([y_min_main_line[i_l], y_min_main_tab[i_t]])) + y_down_tab.append(np.max([y_max_main_line[i_l], y_max_main_tab[i_t]])) + + if len(y_up_tab) == 0: + for v_n in range(len(cx_tab_m_text)): + if cx_tabl[i_t] <= x_max_tab_m_text[v_n] and cx_tabl[i_t] >= x_min_tab_m_text[v_n] and cy_tabl[i_t] <= y_max_tab_m_text[v_n] and cy_tabl[i_t] >= y_min_tab_m_text[v_n] and cx_tabl[i_t] != cx_tab_m_text[v_n] and cy_tabl[i_t] != cy_tab_m_text[v_n]: + y_up_tabs.append(y_min_tab_m_text[v_n]) + y_down_tabs.append(y_max_tab_m_text[v_n]) + # y_up_tabs.append(y_min_main_tab[i_t]) + # y_down_tabs.append(y_max_main_tab[i_t]) + else: + y_up_tabs.append(np.min(y_up_tab)) + y_down_tabs.append(np.max(y_down_tab)) + + else: + y_down_tabs = [] + y_up_tabs = [] + pass + + for ii in range(len(y_up_tabs)): + image_box[y_up_tabs[ii] : y_down_tabs[ii], :, 0] = 7 + + image_revised_last[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][0]) : int(boxes[i][1]), :] = image_box[:, :, :] + + else: + for i in range(len(boxes)): + + image_box = img_comm[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][0]) : int(boxes[i][1]), :] + image_revised_last[int(boxes[i][2]) : int(boxes[i][3]), int(boxes[i][0]) : int(boxes[i][1]), :] = image_box[:, :, :] + + ##plt.figure(figsize=(20,20)) + ##plt.imshow(image_box[:,:,0]) + ##plt.show() + return image_revised_last + +def get_regions_from_xy_2models_ens(self, img): + img_org = np.copy(img) + + img_height_h = img_org.shape[0] + img_width_h = img_org.shape[1] + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) + + gaussian_filter = False + patches = False + binary = False + + ratio_x = 1 + ratio_y = 1 + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + prediction_regions_long = self.do_prediction(patches, img, model_region) + + prediction_regions_long = resize_image(prediction_regions_long, img_height_h, img_width_h) + + gaussian_filter = False + patches = True + binary = False + + ratio_x = 1 + ratio_y = 1.2 + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_org_y = self.do_prediction(patches, img, model_region) + + prediction_regions_org_y = resize_image(prediction_regions_org_y, img_height_h, img_width_h) + + # plt.imshow(prediction_regions_org[:,:,0]) + # plt.show() + # sys.exit() + prediction_regions_org_y = prediction_regions_org_y[:, :, 0] + + mask_zeros_y = (prediction_regions_org_y[:, :] == 0) * 1 + + ratio_x = 1.2 + ratio_y = 1 + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_org = self.do_prediction(patches, img, model_region) + + prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h) + + # plt.imshow(prediction_regions_org[:,:,0]) + # plt.show() + # sys.exit() + prediction_regions_org = prediction_regions_org[:, :, 0] + + prediction_regions_org[(prediction_regions_org[:, :] == 1) & (mask_zeros_y[:, :] == 1)] = 0 + + prediction_regions_org[(prediction_regions_long[:, :, 0] == 1) & (prediction_regions_org[:, :] == 2)] = 1 + + session_region.close() + del model_region + del session_region + gc.collect() + + return prediction_regions_org + +def resize_and_enhance_image(self, is_image_enhanced): + dpi = self.check_dpi() + img = cv2.imread(self.image_dir) + img = img.astype(np.uint8) + # sys.exit() + + print(dpi) + + if dpi < 298: + if img.shape[0] < 1000: + img_h_new = int(img.shape[0] * 3) + img_w_new = int(img.shape[1] * 3) + if img_h_new < 2800: + img_h_new = 3000 + img_w_new = int(img.shape[1] / float(img.shape[0]) * 3000) + elif img.shape[0] >= 1000 and img.shape[0] < 2000: + img_h_new = int(img.shape[0] * 2) + img_w_new = int(img.shape[1] * 2) + if img_h_new < 2800: + img_h_new = 3000 + img_w_new = int(img.shape[1] / float(img.shape[0]) * 3000) + else: + img_h_new = int(img.shape[0] * 1.5) + img_w_new = int(img.shape[1] * 1.5) + img_new = resize_image(img, img_h_new, img_w_new) + image_res = self.predict_enhancement(img_new) + # cv2.imwrite(os.path.join(self.dir_out, self.f_name) + ".tif",self.image) + # self.image=self.image.astype(np.uint16) + + # self.scale_x=1 + # self.scale_y=1 + # self.height_org = self.image.shape[0] + # self.width_org = self.image.shape[1] + is_image_enhanced = True + else: + is_image_enhanced = False + image_res = np.copy(img) + + return is_image_enhanced, img, image_res + +def resize_and_enhance_image_new(self, is_image_enhanced): + # self.check_dpi() + img = cv2.imread(self.image_dir) + img = img.astype(np.uint8) + # sys.exit() + + image_res = np.copy(img) + + return is_image_enhanced, img, image_res + +def get_image_and_scales_deskewd(self, img_deskewd): + + self.image = img_deskewd + self.image_org = np.copy(self.image) + self.height_org = self.image.shape[0] + self.width_org = self.image.shape[1] + + self.img_hight_int = int(self.image.shape[0] * 1) + self.img_width_int = int(self.image.shape[1] * 1) + self.scale_y = self.img_hight_int / float(self.image.shape[0]) + self.scale_x = self.img_width_int / float(self.image.shape[1]) + + self.image = resize_image(self.image, self.img_hight_int, self.img_width_int) + +def extract_drop_capital_13(self, img, patches, cols): + + img_height_h = img.shape[0] + img_width_h = img.shape[1] + patches = False + + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully_np) + + img_1 = img[: int(img.shape[0] / 3.0), :, :] + img_2 = img[int(img.shape[0] / 3.0) : int(2 * img.shape[0] / 3.0), :, :] + img_3 = img[int(2 * img.shape[0] / 3.0) :, :, :] + + # img_1 = otsu_copy_binary(img_1)#otsu_copy(img) + # img_1 = img_1.astype(np.uint16) + + plt.imshow(img_1) + plt.show() + # img_2 = otsu_copy_binary(img_2)#otsu_copy(img) + # img_2 = img_2.astype(np.uint16) + + plt.imshow(img_2) + plt.show() + # img_3 = otsu_copy_binary(img_3)#otsu_copy(img) + # img_3 = img_3.astype(np.uint16) + + plt.imshow(img_3) + plt.show() + + prediction_regions_1 = self.do_prediction(patches, img_1, model_region) + + plt.imshow(prediction_regions_1) + plt.show() + + prediction_regions_2 = self.do_prediction(patches, img_2, model_region) + + plt.imshow(prediction_regions_2) + plt.show() + prediction_regions_3 = self.do_prediction(patches, img_3, model_region) + + plt.imshow(prediction_regions_3) + plt.show() + prediction_regions = np.zeros((img_height_h, img_width_h)) + + prediction_regions[: int(img.shape[0] / 3.0), :] = prediction_regions_1[:, :, 0] + prediction_regions[int(img.shape[0] / 3.0) : int(2 * img.shape[0] / 3.0), :] = prediction_regions_2[:, :, 0] + prediction_regions[int(2 * img.shape[0] / 3.0) :, :] = prediction_regions_3[:, :, 0] + + session_region.close() + del img_1 + del img_2 + del img_3 + del prediction_regions_1 + del prediction_regions_2 + del prediction_regions_3 + del model_region + del session_region + del img + gc.collect() + return prediction_regions + +def extract_only_text_regions(self, img, patches): + + model_region, session_region = self.start_new_session_and_model(self.model_only_text) + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint8) + img_org = np.copy(img) + + img_h = img_org.shape[0] + img_w = img_org.shape[1] + + img = resize_image(img_org, int(img_org.shape[0] * 1), int(img_org.shape[1] * 1)) + + prediction_regions1 = self.do_prediction(patches, img, model_region) + + prediction_regions1 = resize_image(prediction_regions1, img_h, img_w) + + # prediction_regions1 = cv2.dilate(prediction_regions1, self.kernel, iterations=4) + # prediction_regions1 = cv2.erode(prediction_regions1, self.kernel, iterations=7) + # prediction_regions1 = cv2.dilate(prediction_regions1, self.kernel, iterations=2) + + img = resize_image(img_org, int(img_org.shape[0] * 1), int(img_org.shape[1] * 1)) + + prediction_regions2 = self.do_prediction(patches, img, model_region) + + prediction_regions2 = resize_image(prediction_regions2, img_h, img_w) + + # prediction_regions2 = cv2.dilate(prediction_regions2, self.kernel, iterations=2) + prediction_regions2 = cv2.erode(prediction_regions2, self.kernel, iterations=2) + prediction_regions2 = cv2.dilate(prediction_regions2, self.kernel, iterations=2) + + # prediction_regions=( (prediction_regions2[:,:,0]==1) & (prediction_regions1[:,:,0]==1) ) + # prediction_regions=(prediction_regions1[:,:,0]==1) + + session_region.close() + del model_region + del session_region + gc.collect() + return prediction_regions1[:, :, 0] + +def extract_binarization(self, img, patches): + + model_bin, session_bin = self.start_new_session_and_model(self.model_binafrization) + + img_h = img.shape[0] + img_w = img.shape[1] + + img = resize_image(img, int(img.shape[0] * 1), int(img.shape[1] * 1)) + + prediction_regions = self.do_prediction(patches, img, model_bin) + + res = (prediction_regions[:, :, 0] != 0) * 1 + + img_fin = np.zeros((res.shape[0], res.shape[1], 3)) + res[:, :][res[:, :] == 0] = 2 + res = res - 1 + res = res * 255 + img_fin[:, :, 0] = res + img_fin[:, :, 1] = res + img_fin[:, :, 2] = res + + session_bin.close() + del model_bin + del session_bin + gc.collect() + # plt.imshow(img_fin[:,:,0]) + # plt.show() + return img_fin + +def get_text_region_contours_and_boxes(self, image): + rgb_class_of_texts = (1, 1, 1) + mask_texts = np.all(image == rgb_class_of_texts, axis=-1) + + image = np.repeat(mask_texts[:, :, np.newaxis], 3, axis=2) * 255 + image = image.astype(np.uint8) + + image = cv2.morphologyEx(image, cv2.MORPH_OPEN, self.kernel) + image = cv2.morphologyEx(image, cv2.MORPH_CLOSE, self.kernel) + + imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + + _, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours, hirarchy = cv2.findContours(thresh.copy(), cv2.cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + main_contours = filter_contours_area_of_image(thresh, contours, hirarchy, max_area=1, min_area=0.00001) + self.boxes = [] + + for jj in range(len(main_contours)): + x, y, w, h = cv2.boundingRect(main_contours[jj]) + self.boxes.append([x, y, w, h]) + + return main_contours + +def textline_contours_to_get_slope_correctly(self, textline_mask, img_patch, contour_interest): + + slope_new = 0 # deskew_images(img_patch) + + textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 + + textline_mask = textline_mask.astype(np.uint8) + textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_OPEN, self.kernel) + textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, self.kernel) + textline_mask = cv2.erode(textline_mask, self.kernel, iterations=1) + imgray = cv2.cvtColor(textline_mask, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + + thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, self.kernel) + thresh = cv2.morphologyEx(thresh, cv2.MORPH_CLOSE, self.kernel) + + contours, hirarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area=1, min_area=0.003) + + textline_maskt = textline_mask[:, :, 0] + textline_maskt[textline_maskt != 0] = 1 + + peaks_point, _ = seperate_lines(textline_maskt, contour_interest, slope_new) + + mean_dis = np.mean(np.diff(peaks_point)) + + len_x = thresh.shape[1] + + slope_lines = [] + contours_slope_new = [] + + for kk in range(len(main_contours)): + + if len(main_contours[kk].shape) == 2: + xminh = np.min(main_contours[kk][:, 0]) + xmaxh = np.max(main_contours[kk][:, 0]) + + yminh = np.min(main_contours[kk][:, 1]) + ymaxh = np.max(main_contours[kk][:, 1]) + elif len(main_contours[kk].shape) == 3: + xminh = np.min(main_contours[kk][:, 0, 0]) + xmaxh = np.max(main_contours[kk][:, 0, 0]) + + yminh = np.min(main_contours[kk][:, 0, 1]) + ymaxh = np.max(main_contours[kk][:, 0, 1]) + + if ymaxh - yminh <= mean_dis and (xmaxh - xminh) >= 0.3 * len_x: # xminh>=0.05*len_x and xminh<=0.4*len_x and xmaxh<=0.95*len_x and xmaxh>=0.6*len_x: + contours_slope_new.append(main_contours[kk]) + + rows, cols = thresh.shape[:2] + [vx, vy, x, y] = cv2.fitLine(main_contours[kk], cv2.DIST_L2, 0, 0.01, 0.01) + + slope_lines.append((vy / vx) / np.pi * 180) + + if len(slope_lines) >= 2: + + slope = np.mean(slope_lines) # slope_true/np.pi*180 + else: + slope = 999 + + else: + slope = 0 + + return slope + + +def return_deskew_slope_new(self, img_patch, sigma_des): + max_x_y = max(img_patch.shape[0], img_patch.shape[1]) + + ##img_patch=resize_image(img_patch,max_x_y,max_x_y) + + img_patch_copy = np.zeros((img_patch.shape[0], img_patch.shape[1])) + img_patch_copy[:, :] = img_patch[:, :] # img_patch_org[:,:,0] + + img_patch_padded = np.zeros((int(max_x_y * (1.4)), int(max_x_y * (1.4)))) + + img_patch_padded_center_p = int(img_patch_padded.shape[0] / 2.0) + len_x_org_patch_half = int(img_patch_copy.shape[1] / 2.0) + len_y_org_patch_half = int(img_patch_copy.shape[0] / 2.0) + + img_patch_padded[img_patch_padded_center_p - len_y_org_patch_half : img_patch_padded_center_p - len_y_org_patch_half + img_patch_copy.shape[0], img_patch_padded_center_p - len_x_org_patch_half : img_patch_padded_center_p - len_x_org_patch_half + img_patch_copy.shape[1]] = img_patch_copy[:, :] + # img_patch_padded[ int( img_patch_copy.shape[0]*(.1)):int( img_patch_copy.shape[0]*(.1))+img_patch_copy.shape[0] , int( img_patch_copy.shape[1]*(.8)):int( img_patch_copy.shape[1]*(.8))+img_patch_copy.shape[1] ]=img_patch_copy[:,:] + angles = np.linspace(-25, 25, 80) + + res = [] + num_of_peaks = [] + index_cor = [] + var_res = [] + + # plt.imshow(img_patch) + # plt.show() + indexer = 0 + for rot in angles: + # print(rot,'rot') + img_rotated = rotate_image(img_patch_padded, rot) + img_rotated[img_rotated != 0] = 1 + + # plt.imshow(img_rotated) + # plt.show() + + try: + neg_peaks, var_spectrum = self.get_standard_deviation_of_summed_textline_patch_along_width(img_rotated, sigma_des, 20.3) + res_me = np.mean(neg_peaks) + if res_me == 0: + res_me = VERY_LARGE_NUMBER + else: + pass + + res_num = len(neg_peaks) + except: + res_me = VERY_LARGE_NUMBER + res_num = 0 + var_spectrum = 0 + if isNaN(res_me): + pass + else: + res.append(res_me) + var_res.append(var_spectrum) + num_of_peaks.append(res_num) + index_cor.append(indexer) + indexer = indexer + 1 + + try: + var_res = np.array(var_res) + # print(var_res) + + ang_int = angles[np.argmax(var_res)] # angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int = 0 + + if abs(ang_int) > 15: + angles = np.linspace(-90, -50, 30) + res = [] + num_of_peaks = [] + index_cor = [] + var_res = [] + + # plt.imshow(img_patch) + # plt.show() + indexer = 0 + for rot in angles: + # print(rot,'rot') + img_rotated = rotate_image(img_patch_padded, rot) + img_rotated[img_rotated != 0] = 1 + + # plt.imshow(img_rotated) + # plt.show() + + try: + neg_peaks, var_spectrum = self.get_standard_deviation_of_summed_textline_patch_along_width(img_rotated, sigma_des, 20.3) + res_me = np.mean(neg_peaks) + if res_me == 0: + res_me = VERY_LARGE_NUMBER + else: + pass + + res_num = len(neg_peaks) + except: + res_me = VERY_LARGE_NUMBER + res_num = 0 + var_spectrum = 0 + if isNaN(res_me): + pass + else: + res.append(res_me) + var_res.append(var_spectrum) + num_of_peaks.append(res_num) + index_cor.append(indexer) + indexer = indexer + 1 + + try: + var_res = np.array(var_res) + # print(var_res) + + ang_int = angles[np.argmax(var_res)] # angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int = 0 + + return ang_int + +def get_slopes_and_deskew(self, contours, textline_mask_tot): + + slope_biggest = 0 # return_deskew_slop(img_int_p,sigma_des, dir_of_all=self.dir_of_all, f_name=self.f_name) + + num_cores = cpu_count() + q = Queue() + poly = Queue() + box_sub = Queue() + + processes = [] + nh = np.linspace(0, len(self.boxes), num_cores + 1) + + for i in range(num_cores): + boxes_per_process = self.boxes[int(nh[i]) : int(nh[i + 1])] + contours_per_process = contours[int(nh[i]) : int(nh[i + 1])] + processes.append(Process(target=self.do_work_of_slopes, args=(q, poly, box_sub, boxes_per_process, textline_mask_tot, contours_per_process))) + + for i in range(num_cores): + processes[i].start() + + self.slopes = [] + self.all_found_texline_polygons = [] + self.boxes = [] + + for i in range(num_cores): + slopes_for_sub_process = q.get(True) + boxes_for_sub_process = box_sub.get(True) + polys_for_sub_process = poly.get(True) + + for j in range(len(slopes_for_sub_process)): + self.slopes.append(slopes_for_sub_process[j]) + self.all_found_texline_polygons.append(polys_for_sub_process[j]) + self.boxes.append(boxes_for_sub_process[j]) + + for i in range(num_cores): + processes[i].join() + + +def write_into_page_xml_only_textlines(self, contours, page_coord, all_found_texline_polygons, all_box_coord, dir_of_image): + + found_polygons_text_region = contours + + # create the file structure + data = ET.Element("PcGts") + + data.set("xmlns", "http://schema.primaresearch.org/PAGE/gts/pagecontent/2017-07-15") + data.set("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance") + data.set("xsi:schemaLocation", "http://schema.primaresearch.org/PAGE/gts/pagecontent/2017-07-15") + + metadata = ET.SubElement(data, "Metadata") + + author = ET.SubElement(metadata, "Creator") + author.text = "SBB_QURATOR" + + created = ET.SubElement(metadata, "Created") + created.text = "2019-06-17T18:15:12" + + changetime = ET.SubElement(metadata, "LastChange") + changetime.text = "2019-06-17T18:15:12" + + page = ET.SubElement(data, "Page") + + page.set("imageFilename", self.image_dir) + page.set("imageHeight", str(self.height_org)) + page.set("imageWidth", str(self.width_org)) + page.set("type", "content") + page.set("readingDirection", "left-to-right") + page.set("textLineOrder", "top-to-bottom") + + page_print_sub = ET.SubElement(page, "PrintSpace") + coord_page = ET.SubElement(page_print_sub, "Coords") + points_page_print = "" + + for lmm in range(len(self.cont_page[0])): + if len(self.cont_page[0][lmm]) == 2: + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0]) / self.scale_x)) + points_page_print = points_page_print + "," + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][1]) / self.scale_y)) + else: + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0][0]) / self.scale_x)) + points_page_print = points_page_print + "," + points_page_print = points_page_print + str(int((self.cont_page[0][lmm][0][1]) / self.scale_y)) + + if lmm < (len(self.cont_page[0]) - 1): + points_page_print = points_page_print + " " + coord_page.set("points", points_page_print) + + if len(contours) > 0: + + id_indexer = 0 + id_indexer_l = 0 + + for mm in range(len(found_polygons_text_region)): + textregion = ET.SubElement(page, "TextRegion") + + textregion.set("id", "r" + str(id_indexer)) + id_indexer += 1 + + textregion.set("type", "paragraph") + # if mm==0: + # textregion.set('type','header') + # else: + # textregion.set('type','paragraph') + coord_text = ET.SubElement(textregion, "Coords") + + points_co = "" + for lmm in range(len(found_polygons_text_region[mm])): + if len(found_polygons_text_region[mm][lmm]) == 2: + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((found_polygons_text_region[mm][lmm][0][1] + page_coord[0]) / self.scale_y)) + + if lmm < (len(found_polygons_text_region[mm]) - 1): + points_co = points_co + " " + # print(points_co) + coord_text.set("points", points_co) + + for j in range(len(all_found_texline_polygons[mm])): + + textline = ET.SubElement(textregion, "TextLine") + + textline.set("id", "l" + str(id_indexer_l)) + + id_indexer_l += 1 + + coord = ET.SubElement(textline, "Coords") + + texteq = ET.SubElement(textline, "TextEquiv") + + uni = ET.SubElement(texteq, "Unicode") + uni.text = " " + + # points = ET.SubElement(coord, 'Points') + + points_co = "" + for l in range(len(all_found_texline_polygons[mm][j])): + # point = ET.SubElement(coord, 'Point') + + # point.set('x',str(found_polygons[j][l][0])) + # point.set('y',str(found_polygons[j][l][1])) + if len(all_found_texline_polygons[mm][j][l]) == 2: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][1] + page_coord[0]) / self.scale_y)) + else: + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co = points_co + "," + points_co = points_co + str(int((all_found_texline_polygons[mm][j][l][0][1] + page_coord[0]) / self.scale_y)) + + if l < (len(all_found_texline_polygons[mm][j]) - 1): + points_co = points_co + " " + # print(points_co) + coord.set("points", points_co) + + texteqreg = ET.SubElement(textregion, "TextEquiv") + + unireg = ET.SubElement(texteqreg, "Unicode") + unireg.text = " " + + # print(dir_of_image) + print(self.f_name) + # print(os.path.join(dir_of_image, self.f_name) + ".xml") + tree = ET.ElementTree(data) + tree.write(os.path.join(dir_of_image, self.f_name) + ".xml") + +def return_teilwiese_deskewed_lines(self, text_regions_p, textline_rotated): + + kernel = np.ones((5, 5), np.uint8) + textline_rotated = cv2.erode(textline_rotated, kernel, iterations=1) + + textline_rotated_new = np.zeros(textline_rotated.shape) + rgb_m = 1 + rgb_h = 2 + + cnt_m, boxes_m = return_contours_of_interested_region_and_bounding_box(text_regions_p, rgb_m) + cnt_h, boxes_h = return_contours_of_interested_region_and_bounding_box(text_regions_p, rgb_h) + + areas_cnt_m = np.array([cv2.contourArea(cnt_m[j]) for j in range(len(cnt_m))]) + + argmax = np.argmax(areas_cnt_m) + + # plt.imshow(textline_rotated[ boxes_m[argmax][1]:boxes_m[argmax][1]+boxes_m[argmax][3] ,boxes_m[argmax][0]:boxes_m[argmax][0]+boxes_m[argmax][2]]) + # plt.show() + + for argmax in range(len(boxes_m)): + + textline_text_region = textline_rotated[boxes_m[argmax][1] : boxes_m[argmax][1] + boxes_m[argmax][3], boxes_m[argmax][0] : boxes_m[argmax][0] + boxes_m[argmax][2]] + + textline_text_region_revised = self.seperate_lines_new(textline_text_region, 0) + # except: + # textline_text_region_revised=textline_rotated[ boxes_m[argmax][1]:boxes_m[argmax][1]+boxes_m[argmax][3] ,boxes_m[argmax][0]:boxes_m[argmax][0]+boxes_m[argmax][2] ] + textline_rotated_new[boxes_m[argmax][1] : boxes_m[argmax][1] + boxes_m[argmax][3], boxes_m[argmax][0] : boxes_m[argmax][0] + boxes_m[argmax][2]] = textline_text_region_revised[:, :] + + # textline_rotated_new[textline_rotated_new>0]=1 + # textline_rotated_new[textline_rotated_new<0]=0 + # plt.imshow(textline_rotated_new) + # plt.show() + +def get_regions_from_xy_neu(self, img): + img_org = np.copy(img) + + img_height_h = img_org.shape[0] + img_width_h = img_org.shape[1] + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p) + + gaussian_filter = False + patches = True + binary = True + + ratio_x = 1 + ratio_y = 1 + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_org = self.do_prediction(patches, img, model_region) + + prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h) + + # plt.imshow(prediction_regions_org[:,:,0]) + # plt.show() + # sys.exit() + prediction_regions_org = prediction_regions_org[:, :, 0] + + gaussian_filter = False + patches = False + binary = False + + ratio_x = 1 + ratio_y = 1 + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_orgt = self.do_prediction(patches, img, model_region) + + prediction_regions_orgt = resize_image(prediction_regions_orgt, img_height_h, img_width_h) + + # plt.imshow(prediction_regions_orgt[:,:,0]) + # plt.show() + # sys.exit() + prediction_regions_orgt = prediction_regions_orgt[:, :, 0] + + mask_texts_longshot = (prediction_regions_orgt[:, :] == 1) * 1 + + mask_texts_longshot = np.uint8(mask_texts_longshot) + # mask_texts_longshot = cv2.dilate(mask_texts_longshot[:,:], self.kernel, iterations=2) + + pixel_img = 1 + polygons_of_only_texts_longshot = return_contours_of_interested_region(mask_texts_longshot, pixel_img) + + longshot_true = np.zeros(mask_texts_longshot.shape) + # text_regions_p_true[:,:]=text_regions_p_1[:,:] + + longshot_true = cv2.fillPoly(longshot_true, pts=polygons_of_only_texts_longshot, color=(1, 1, 1)) + + # plt.imshow(longshot_true) + # plt.show() + + gaussian_filter = False + patches = False + binary = False + + ratio_x = 1 + ratio_y = 1 + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + one_third_upper_ny = int(img.shape[0] / 3.0) + + img = img[0:one_third_upper_ny, :, :] + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_longshot_one_third = self.do_prediction(patches, img, model_region) + + prediction_regions_longshot_one_third = resize_image(prediction_regions_longshot_one_third, one_third_upper_ny, img_width_h) + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + img = img[one_third_upper_ny : int(2 * one_third_upper_ny), :, :] + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_longshot_one_third_middle = self.do_prediction(patches, img, model_region) + + prediction_regions_longshot_one_third_middle = resize_image(prediction_regions_longshot_one_third_middle, one_third_upper_ny, img_width_h) + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + img = img[int(2 * one_third_upper_ny) :, :, :] + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_longshot_one_third_down = self.do_prediction(patches, img, model_region) + + prediction_regions_longshot_one_third_down = resize_image(prediction_regions_longshot_one_third_down, img_height_h - int(2 * one_third_upper_ny), img_width_h) + + # plt.imshow(prediction_regions_org[:,:,0]) + # plt.show() + # sys.exit() + prediction_regions_longshot = np.zeros((img_height_h, img_width_h)) + + # prediction_regions_longshot=prediction_regions_longshot[:,:,0] + + # prediction_regions_longshot[0:one_third_upper_ny,:]=prediction_regions_longshot_one_third[:,:,0] + # prediction_regions_longshot[one_third_upper_ny:int(2*one_third_upper_ny):,:]=prediction_regions_longshot_one_third_middle[:,:,0] + # prediction_regions_longshot[int(2*one_third_upper_ny):,:]=prediction_regions_longshot_one_third_down[:,:,0] + + prediction_regions_longshot = longshot_true[:, :] + # plt.imshow(prediction_regions_longshot) + # plt.show() + + gaussian_filter = False + patches = True + binary = False + + ratio_x = 1 # 1.1 + ratio_y = 1 + median_blur = False + + # img= resize_image(img_org, int(img_org.shape[0]*0.8), int(img_org.shape[1]*1.6)) + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + + prediction_regions = self.do_prediction(patches, img, model_region) + text_region1 = resize_image(prediction_regions, img_height_h, img_width_h) + + # plt.imshow(text_region1[:,:,0]) + # plt.show() + ratio_x = 1 + ratio_y = 1.2 # 1.3 + binary = False + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + + prediction_regions = self.do_prediction(patches, img, model_region) + text_region2 = resize_image(prediction_regions, img_height_h, img_width_h) + + # plt.imshow(text_region2[:,:,0]) + # plt.show() + session_region.close() + del model_region + del session_region + gc.collect() + + # text_region1=text_region1[:,:,0] + # text_region2=text_region2[:,:,0] + + # text_region1[(text_region1[:,:]==2) & (text_region2[:,:]==1)]=1 + + mask_zeros_from_1 = (text_region2[:, :, 0] == 0) * 1 + # mask_text_from_1=(text_region1[:,:,0]==1)*1 + + mask_img_text_region1 = (text_region1[:, :, 0] == 2) * 1 + text_region2_1st_channel = text_region1[:, :, 0] + + text_region2_1st_channel[mask_zeros_from_1 == 1] = 0 + + ##text_region2_1st_channel[mask_img_text_region1[:,:]==1]=2 + # text_region2_1st_channel[(mask_text_from_1==1) & (text_region2_1st_channel==2)]=1 + + mask_lines1 = (text_region1[:, :, 0] == 3) * 1 + mask_lines2 = (text_region2[:, :, 0] == 3) * 1 + + mask_lines2[mask_lines1[:, :] == 1] = 1 + + # plt.imshow(text_region2_1st_channel) + # plt.show() + + text_region2_1st_channel = cv2.erode(text_region2_1st_channel[:, :], self.kernel, iterations=4) + + # plt.imshow(text_region2_1st_channel) + # plt.show() + + text_region2_1st_channel = cv2.dilate(text_region2_1st_channel[:, :], self.kernel, iterations=4) + + text_region2_1st_channel[mask_lines2[:, :] == 1] = 3 + + # text_region2_1st_channel[ (prediction_regions_org[:,:]==1) & (text_region2_1st_channel[:,:]==2)]=1 + + # only in the case of model 3 + + text_region2_1st_channel[(prediction_regions_longshot[:, :] == 1) & (text_region2_1st_channel[:, :] == 2)] = 1 + + text_region2_1st_channel[(prediction_regions_org[:, :] == 2) & (text_region2_1st_channel[:, :] == 0)] = 2 + + # text_region2_1st_channel[prediction_regions_org[:,:]==0]=0 + + # plt.imshow(text_region2_1st_channel) + # plt.show() + + # text_region2_1st_channel[:,:400]=0 + + mask_texts_only = (text_region2_1st_channel[:, :] == 1) * 1 + + mask_images_only = (text_region2_1st_channel[:, :] == 2) * 1 + + mask_lines_only = (text_region2_1st_channel[:, :] == 3) * 1 + + pixel_img = 1 + polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only, pixel_img) + + polygons_of_only_images = return_contours_of_interested_region(mask_images_only, pixel_img) + + polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only, pixel_img) + + text_regions_p_true = np.zeros(text_region2_1st_channel.shape) + # text_regions_p_true[:,:]=text_regions_p_1[:,:] + + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_lines, color=(3, 3, 3)) + + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_images, color=(2, 2, 2)) + + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_texts, color=(1, 1, 1)) + + ##print(np.unique(text_regions_p_true)) + + # text_regions_p_true_3d=np.repeat(text_regions_p_1[:, :, np.newaxis], 3, axis=2) + # text_regions_p_true_3d=text_regions_p_true_3d.astype(np.uint8) + + return text_regions_p_true # text_region2_1st_channel + +def get_regions_from_xy(self, img): + img_org = np.copy(img) + + img_height_h = img_org.shape[0] + img_width_h = img_org.shape[1] + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p) + + gaussian_filter = False + patches = True + binary = True + + ratio_x = 1 + ratio_y = 1 + median_blur = False + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + prediction_regions_org = self.do_prediction(patches, img, model_region) + + ###plt.imshow(prediction_regions_org[:,:,0]) + ###plt.show() + ##sys.exit() + prediction_regions_org = prediction_regions_org[:, :, 0] + + gaussian_filter = False + patches = True + binary = False + + ratio_x = 1.1 + ratio_y = 1 + median_blur = False + + # img= resize_image(img_org, int(img_org.shape[0]*0.8), int(img_org.shape[1]*1.6)) + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + + prediction_regions = self.do_prediction(patches, img, model_region) + text_region1 = resize_image(prediction_regions, img_height_h, img_width_h) + + ratio_x = 1 + ratio_y = 1.1 + binary = False + median_blur = False + + img = resize_image(img_org, int(img_org.shape[0] * ratio_y), int(img_org.shape[1] * ratio_x)) + + if binary: + img = otsu_copy_binary(img) # otsu_copy(img) + img = img.astype(np.uint16) + + if median_blur: + img = cv2.medianBlur(img, 5) + if gaussian_filter: + img = cv2.GaussianBlur(img, (5, 5), 0) + img = img.astype(np.uint16) + + prediction_regions = self.do_prediction(patches, img, model_region) + text_region2 = resize_image(prediction_regions, img_height_h, img_width_h) + + session_region.close() + del model_region + del session_region + gc.collect() + + mask_zeros_from_1 = (text_region1[:, :, 0] == 0) * 1 + # mask_text_from_1=(text_region1[:,:,0]==1)*1 + + mask_img_text_region1 = (text_region1[:, :, 0] == 2) * 1 + text_region2_1st_channel = text_region2[:, :, 0] + + text_region2_1st_channel[mask_zeros_from_1 == 1] = 0 + + text_region2_1st_channel[mask_img_text_region1[:, :] == 1] = 2 + # text_region2_1st_channel[(mask_text_from_1==1) & (text_region2_1st_channel==2)]=1 + + mask_lines1 = (text_region1[:, :, 0] == 3) * 1 + mask_lines2 = (text_region2[:, :, 0] == 3) * 1 + + mask_lines2[mask_lines1[:, :] == 1] = 1 + + ##plt.imshow(text_region2_1st_channel) + ##plt.show() + + text_region2_1st_channel = cv2.erode(text_region2_1st_channel[:, :], self.kernel, iterations=5) + + ##plt.imshow(text_region2_1st_channel) + ##plt.show() + + text_region2_1st_channel = cv2.dilate(text_region2_1st_channel[:, :], self.kernel, iterations=5) + + text_region2_1st_channel[mask_lines2[:, :] == 1] = 3 + + text_region2_1st_channel[(prediction_regions_org[:, :] == 1) & (text_region2_1st_channel[:, :] == 2)] = 1 + text_region2_1st_channel[prediction_regions_org[:, :] == 3] = 3 + + ##plt.imshow(text_region2_1st_channel) + ##plt.show() + return text_region2_1st_channel + +def do_work_of_textline_seperation(self, queue_of_all_params, polygons_per_process, index_polygons_per_process, con_par_org, textline_mask_tot, mask_texts_only, num_col, scale_par, boxes_text): + + textregions_cnt_tot_per_process = [] + textlines_cnt_tot_per_process = [] + index_polygons_per_process_per_process = [] + polygons_per_par_process_per_process = [] + textline_cnt_seperated = np.zeros(textline_mask_tot.shape) + for iiii in range(len(polygons_per_process)): + # crop_img,crop_coor=crop_image_inside_box(boxes_text[mv],image_page_rotated) + # arg_max=np.argmax(areas_cnt_only_text) + textregions_cnt_tot_per_process.append(polygons_per_process[iiii] / scale_par) + textline_region_in_image = np.zeros(textline_mask_tot.shape) + cnt_o_t_max = polygons_per_process[iiii] + + x, y, w, h = cv2.boundingRect(cnt_o_t_max) + + mask_biggest = np.zeros(mask_texts_only.shape) + mask_biggest = cv2.fillPoly(mask_biggest, pts=[cnt_o_t_max], color=(1, 1, 1)) + + mask_region_in_patch_region = mask_biggest[y : y + h, x : x + w] + + textline_biggest_region = mask_biggest * textline_mask_tot + + textline_rotated_seperated = self.seperate_lines_new2(textline_biggest_region[y : y + h, x : x + w], 0, num_col) + + # new line added + ##print(np.shape(textline_rotated_seperated),np.shape(mask_biggest)) + textline_rotated_seperated[mask_region_in_patch_region[:, :] != 1] = 0 + # till here + + textline_cnt_seperated[y : y + h, x : x + w] = textline_rotated_seperated + textline_region_in_image[y : y + h, x : x + w] = textline_rotated_seperated + + # plt.imshow(textline_region_in_image) + # plt.show() + + # plt.imshow(textline_cnt_seperated) + # plt.show() + + pixel_img = 1 + cnt_textlines_in_image = return_contours_of_interested_textline(textline_region_in_image, pixel_img) + + textlines_cnt_per_region = [] + for jjjj in range(len(cnt_textlines_in_image)): + mask_biggest2 = np.zeros(mask_texts_only.shape) + mask_biggest2 = cv2.fillPoly(mask_biggest2, pts=[cnt_textlines_in_image[jjjj]], color=(1, 1, 1)) + if num_col + 1 == 1: + mask_biggest2 = cv2.dilate(mask_biggest2, self.kernel, iterations=5) + else: + + mask_biggest2 = cv2.dilate(mask_biggest2, self.kernel, iterations=4) + + pixel_img = 1 + cnt_textlines_in_image_ind = return_contours_of_interested_textline(mask_biggest2, pixel_img) + + try: + textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0] / scale_par) + except: + pass + # print(len(cnt_textlines_in_image_ind)) + + # plt.imshow(mask_biggest2) + # plt.show() + textlines_cnt_tot_per_process.append(textlines_cnt_per_region) + index_polygons_per_process_per_process.append(index_polygons_per_process[iiii]) + polygons_per_par_process_per_process.append(con_par_org[iiii]) + + queue_of_all_params.put([index_polygons_per_process_per_process, polygons_per_par_process_per_process, textregions_cnt_tot_per_process, textlines_cnt_tot_per_process]) + + +def seperate_lines_new(img_path, thetha, num_col, dir_of_all, f_name): + + if num_col == 1: + num_patches = int(img_path.shape[1] / 200.0) + else: + num_patches = int(img_path.shape[1] / 100.0) + # num_patches=int(img_path.shape[1]/200.) + if num_patches == 0: + num_patches = 1 + (h, w) = img_path.shape[:2] + center = (w // 2, h // 2) + M = cv2.getRotationMatrix2D(center, -thetha, 1.0) + x_d = M[0, 2] + y_d = M[1, 2] + + thetha = thetha / 180.0 * np.pi + rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + + x_min_cont = 0 + x_max_cont = img_path.shape[1] + y_min_cont = 0 + y_max_cont = img_path.shape[0] + + xv = np.linspace(x_min_cont, x_max_cont, 1000) + + mada_n = img_path.sum(axis=1) + + ##plt.plot(mada_n) + ##plt.show() + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + + y = mada_n[:] # [first_nonzero:last_nonzero] + y_help = np.zeros(len(y) + 40) + y_help[20 : len(y) + 20] = y + x = np.array(range(len(y))) + + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + if len(peaks_real) <= 2 and len(peaks_real) > 1: + sigma_gaus = 10 + else: + sigma_gaus = 6 + + z = gaussian_filter1d(y_help, sigma_gaus) + zneg_rev = -y_help + np.max(y_help) + zneg = np.zeros(len(zneg_rev) + 40) + zneg[20 : len(zneg_rev) + 20] = zneg_rev + zneg = gaussian_filter1d(zneg, sigma_gaus) + + peaks, _ = find_peaks(z, height=0) + peaks_neg, _ = find_peaks(zneg, height=0) + + for nn in range(len(peaks_neg)): + if peaks_neg[nn] > len(z) - 1: + peaks_neg[nn] = len(z) - 1 + if peaks_neg[nn] < 0: + peaks_neg[nn] = 0 + + diff_peaks = np.abs(np.diff(peaks_neg)) + cut_off = 20 + peaks_neg_true = [] + forest = [] + + for i in range(len(peaks_neg)): + if i == 0: + forest.append(peaks_neg[i]) + if i < (len(peaks_neg) - 1): + if diff_peaks[i] <= cut_off: + forest.append(peaks_neg[i + 1]) + if diff_peaks[i] > cut_off: + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmin(z[forest])]): + # print(len(z),forest) + peaks_neg_true.append(forest[np.argmin(z[forest])]) + forest = [] + forest.append(peaks_neg[i + 1]) + if i == (len(peaks_neg) - 1): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmin(z[forest])]): + + peaks_neg_true.append(forest[np.argmin(z[forest])]) + + peaks_neg_true = np.array(peaks_neg_true) + + """ + #plt.figure(figsize=(40,40)) + #plt.subplot(1,2,1) + #plt.title('Textline segmentation von Textregion') + #plt.imshow(img_path) + #plt.xlabel('X') + #plt.ylabel('Y') + #plt.subplot(1,2,2) + #plt.title('Dichte entlang X') + #base = pyplot.gca().transData + #rot = transforms.Affine2D().rotate_deg(90) + #plt.plot(zneg,np.array(range(len(zneg)))) + #plt.plot(zneg[peaks_neg_true],peaks_neg_true,'*') + #plt.gca().invert_yaxis() + + #plt.xlabel('Dichte') + #plt.ylabel('Y') + ##plt.plot([0,len(y)], [grenze,grenze]) + #plt.show() + """ + + peaks_neg_true = peaks_neg_true - 20 - 20 + peaks = peaks - 20 + + # dis_up=peaks_neg_true[14]-peaks_neg_true[0] + # dis_down=peaks_neg_true[18]-peaks_neg_true[14] + + img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[15]+dis_down ,:] + + ##plt.imshow(img_patch_ineterst) + ##plt.show() + + length_x = int(img_path.shape[1] / float(num_patches)) + margin = int(0.04 * length_x) + + width_mid = length_x - 2 * margin + + nxf = img_path.shape[1] / float(width_mid) + + if nxf > int(nxf): + nxf = int(nxf) + 1 + else: + nxf = int(nxf) + + slopes_tile_wise = [] + for i in range(nxf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + elif i > 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + + if index_x_u > img_path.shape[1]: + index_x_u = img_path.shape[1] + index_x_d = img_path.shape[1] - length_x + + # img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + + sigma = 2 + try: + slope_xline = return_deskew_slop(img_xline, sigma, dir_of_all=dir_of_all, f_name=f_name) + except: + slope_xline = 0 + slopes_tile_wise.append(slope_xline) + # print(slope_xline,'xlineeee') + img_line_rotated = rotate_image(img_xline, slope_xline) + img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 + + """ + + xline=np.linspace(0,img_path.shape[1],nx) + slopes_tile_wise=[] + + for ui in range( nx-1 ): + img_xline=img_patch_ineterst[:,int(xline[ui]):int(xline[ui+1])] + + + ##plt.imshow(img_xline) + ##plt.show() + + sigma=3 + try: + slope_xline=return_deskew_slop(img_xline,sigma, dir_of_all=self.dir_of_all, f_name=self.f_name) + except: + slope_xline=0 + slopes_tile_wise.append(slope_xline) + print(slope_xline,'xlineeee') + img_line_rotated=rotate_image(img_xline,slope_xline) + + ##plt.imshow(img_line_rotated) + ##plt.show() + """ + + # dis_up=peaks_neg_true[14]-peaks_neg_true[0] + # dis_down=peaks_neg_true[18]-peaks_neg_true[14] + + img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] + + img_patch_ineterst_revised = np.zeros(img_patch_ineterst.shape) + + for i in range(nxf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + elif i > 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + + if index_x_u > img_path.shape[1]: + index_x_u = img_path.shape[1] + index_x_d = img_path.shape[1] - length_x + + img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + + img_int = np.zeros((img_xline.shape[0], img_xline.shape[1])) + img_int[:, :] = img_xline[:, :] # img_patch_org[:,:,0] + + img_resized = np.zeros((int(img_int.shape[0] * (1.2)), int(img_int.shape[1] * (3)))) + + img_resized[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], int(img_int.shape[1] * (1)) : int(img_int.shape[1] * (1)) + img_int.shape[1]] = img_int[:, :] + ##plt.imshow(img_xline) + ##plt.show() + img_line_rotated = rotate_image(img_resized, slopes_tile_wise[i]) + img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 + + img_patch_seperated = seperate_lines_new_inside_teils(img_line_rotated, 0) + + ##plt.imshow(img_patch_seperated) + ##plt.show() + img_patch_seperated_returned = rotate_image(img_patch_seperated, -slopes_tile_wise[i]) + img_patch_seperated_returned[:, :][img_patch_seperated_returned[:, :] != 0] = 1 + + img_patch_seperated_returned_true_size = img_patch_seperated_returned[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], int(img_int.shape[1] * (1)) : int(img_int.shape[1] * (1)) + img_int.shape[1]] + + img_patch_seperated_returned_true_size = img_patch_seperated_returned_true_size[:, margin : length_x - margin] + img_patch_ineterst_revised[:, index_x_d + margin : index_x_u - margin] = img_patch_seperated_returned_true_size + + """ + for ui in range( nx-1 ): + img_xline=img_patch_ineterst[:,int(xline[ui]):int(xline[ui+1])] + + + img_int=np.zeros((img_xline.shape[0],img_xline.shape[1])) + img_int[:,:]=img_xline[:,:]#img_patch_org[:,:,0] + + img_resized=np.zeros((int( img_int.shape[0]*(1.2) ) , int( img_int.shape[1]*(3) ) )) + + img_resized[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ]=img_int[:,:] + ##plt.imshow(img_xline) + ##plt.show() + img_line_rotated=rotate_image(img_resized,slopes_tile_wise[ui]) + + + #img_patch_seperated = seperate_lines_new_inside_teils(img_line_rotated,0) + + img_patch_seperated = seperate_lines_new_inside_teils(img_line_rotated,0) + + img_patch_seperated_returned=rotate_image(img_patch_seperated,-slopes_tile_wise[ui]) + ##plt.imshow(img_patch_seperated) + ##plt.show() + print(img_patch_seperated_returned.shape) + #plt.imshow(img_patch_seperated_returned[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ]) + #plt.show() + + img_patch_ineterst_revised[:,int(xline[ui]):int(xline[ui+1])]=img_patch_seperated_returned[ int( img_int.shape[0]*(.1)):int( img_int.shape[0]*(.1))+img_int.shape[0] , int( img_int.shape[1]*(1)):int( img_int.shape[1]*(1))+img_int.shape[1] ] + + + """ + + # print(img_patch_ineterst_revised.shape,np.unique(img_patch_ineterst_revised)) + ##plt.imshow(img_patch_ineterst_revised) + ##plt.show() + return img_patch_ineterst_revised diff --git a/sbb_newspapers_org_image/utils/__init__.py b/sbb_newspapers_org_image/utils/__init__.py new file mode 100644 index 0000000..d7d1e16 --- /dev/null +++ b/sbb_newspapers_org_image/utils/__init__.py @@ -0,0 +1,2091 @@ +import math + +import matplotlib.pyplot as plt +import numpy as np +from shapely import geometry +import cv2 +import imutils +from scipy.signal import find_peaks +from scipy.ndimage import gaussian_filter1d + +from .is_nan import isNaN +from .contour import (contours_in_same_horizon, + find_new_features_of_contoures, + return_contours_of_image, + return_parent_contours) + + +def crop_image_inside_box(box, img_org_copy): + image_box = img_org_copy[box[1] : box[1] + box[3], box[0] : box[0] + box[2]] + return image_box, [box[1], box[1] + box[3], box[0], box[0] + box[2]] + +def otsu_copy(img): + img_r = np.zeros(img.shape) + img1 = img[:, :, 0] + img2 = img[:, :, 1] + img3 = img[:, :, 2] + # print(img.min()) + # print(img[:,:,0].min()) + # blur = cv2.GaussianBlur(img,(5,5)) + # ret3,th3 = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU) + retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + retval2, threshold2 = cv2.threshold(img2, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + retval3, threshold3 = cv2.threshold(img3, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + + img_r[:, :, 0] = threshold1 + img_r[:, :, 1] = threshold1 + img_r[:, :, 2] = threshold1 + return img_r + +def otsu_copy_binary(img): + img_r = np.zeros((img.shape[0], img.shape[1], 3)) + img1 = img[:, :, 0] + + retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + + img_r[:, :, 0] = threshold1 + img_r[:, :, 1] = threshold1 + img_r[:, :, 2] = threshold1 + + img_r = img_r / float(np.max(img_r)) * 255 + return img_r + + +def find_features_of_lines(contours_main): + + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + + slope_lines = [] + + for kk in range(len(contours_main)): + [vx, vy, x, y] = cv2.fitLine(contours_main[kk], cv2.DIST_L2, 0, 0.01, 0.01) + slope_lines.append(((vy / vx) / np.pi * 180)[0]) + + slope_lines_org = slope_lines + slope_lines = np.array(slope_lines) + slope_lines[(slope_lines < 10) & (slope_lines > -10)] = 0 + + slope_lines[(slope_lines < -200) | (slope_lines > 200)] = 1 + slope_lines[(slope_lines != 0) & (slope_lines != 1)] = 2 + + dis_x = np.abs(x_max_main - x_min_main) + return slope_lines, dis_x, x_min_main, x_max_main, np.array(cy_main), np.array(slope_lines_org), y_min_main, y_max_main, np.array(cx_main) + +def boosting_headers_by_longshot_region_segmentation(textregion_pre_p, textregion_pre_np, img_only_text): + textregion_pre_p_org = np.copy(textregion_pre_p) + # 4 is drop capitals + headers_in_longshot = (textregion_pre_np[:, :, 0] == 2) * 1 + # headers_in_longshot= ( (textregion_pre_np[:,:,0]==2) | (textregion_pre_np[:,:,0]==1) )*1 + textregion_pre_p[:, :, 0][(headers_in_longshot[:, :] == 1) & (textregion_pre_p[:, :, 0] != 4)] = 2 + textregion_pre_p[:, :, 0][textregion_pre_p[:, :, 0] == 1] = 0 + # textregion_pre_p[:,:,0][( img_only_text[:,:]==1) & (textregion_pre_p[:,:,0]!=7) & (textregion_pre_p[:,:,0]!=2)]=1 # eralier it was so, but by this manner the drop capitals are alse deleted + textregion_pre_p[:, :, 0][(img_only_text[:, :] == 1) & (textregion_pre_p[:, :, 0] != 7) & (textregion_pre_p[:, :, 0] != 4) & (textregion_pre_p[:, :, 0] != 2)] = 1 + return textregion_pre_p + + +def find_num_col_deskew(regions_without_seperators, sigma_, multiplier=3.8): + regions_without_seperators_0=regions_without_seperators[:,:].sum(axis=1) + + ##meda_n_updown=regions_without_seperators_0[len(regions_without_seperators_0)::-1] + + ##first_nonzero=(next((i for i, x in enumerate(regions_without_seperators_0) if x), 0)) + ##last_nonzero=(next((i for i, x in enumerate(meda_n_updown) if x), 0)) + + ##last_nonzero=len(regions_without_seperators_0)-last_nonzero + + + y=regions_without_seperators_0#[first_nonzero:last_nonzero] + + ##y_help=np.zeros(len(y)+20) + + ##y_help[10:len(y)+10]=y + + ##x=np.array( range(len(y)) ) + + + + + ##zneg_rev=-y_help+np.max(y_help) + + ##zneg=np.zeros(len(zneg_rev)+20) + + ##zneg[10:len(zneg_rev)+10]=zneg_rev + + z=gaussian_filter1d(y, sigma_) + ###zneg= gaussian_filter1d(zneg, sigma_) + + + ###peaks_neg, _ = find_peaks(zneg, height=0) + ###peaks, _ = find_peaks(z, height=0) + + ###peaks_neg=peaks_neg-10-10 + + ####print(np.std(z),'np.std(z)np.std(z)np.std(z)') + + #####plt.plot(z) + #####plt.show() + + #####plt.imshow(regions_without_seperators) + #####plt.show() + ###""" + ###last_nonzero=last_nonzero-0#100 + ###first_nonzero=first_nonzero+0#+100 + + ###peaks_neg=peaks_neg[(peaks_neg>first_nonzero) & (peaks_neg.06*regions_without_seperators.shape[1]) & (peaks<0.94*regions_without_seperators.shape[1])] + ###""" + ###interest_pos=z[peaks] + + ###interest_pos=interest_pos[interest_pos>10] + + ###interest_neg=z[peaks_neg] + + ###min_peaks_pos=np.mean(interest_pos) + ###min_peaks_neg=0#np.min(interest_neg) + + ###dis_talaei=(min_peaks_pos-min_peaks_neg)/multiplier + ####print(interest_pos) + ###grenze=min_peaks_pos-dis_talaei#np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 + + ###interest_neg_fin=interest_neg[(interest_neg 0)] + max_h_neg_n = max_h[max_h > 0] + + if len(max_h_neg_n) > 0: + max_h_neg = [max_h_neg[np.argmin(max_h_neg_n)]] + else: + max_h_neg = [] + + if len(min_h_neg) > 0 and len(max_h_neg) > 0: + deletions = list(range(min_h_neg[0] + 1, max_h_neg[0])) + unique_delets_int = [] + # print(deletions,len(deletions),'delii') + if len(deletions) > 0: + + for j in range(len(deletions)): + indexes_to_delete.append(deletions[j]) + # print(deletions,indexes_to_delete,'badiii') + unique_delets = np.unique(indexes_to_delete) + # print(min_h_neg[0],unique_delets) + unique_delets_int = unique_delets[unique_delets < min_h_neg[0]] + + indexer_lines_deletions_len.append(len(deletions)) + indexr_uniq_ind.append([deletions]) + + else: + indexer_lines_deletions_len.append(0) + indexr_uniq_ind.append(-999) + + index_line_true = min_h_neg[0] - len(unique_delets_int) + # print(index_line_true) + if index_line_true > 0 and min_h_neg[0] >= 2: + index_line_true = index_line_true + else: + index_line_true = min_h_neg[0] + + indexer_lines.append(index_line_true) + + if len(unique_delets_int) > 0: + for dd in range(len(unique_delets_int)): + indexes_to_delete.append(unique_delets_int[dd]) + else: + indexer_lines.append(-999) + indexer_lines_deletions_len.append(-999) + indexr_uniq_ind.append(-999) + + peaks_true = [] + for m in range(len(peaks_neg_fin_t)): + if m in indexes_to_delete: + pass + else: + peaks_true.append(peaks_neg_fin_t[m]) + return indexer_lines, peaks_true, arg_min_hor_sort, indexer_lines_deletions_len, indexr_uniq_ind + +def find_num_col(regions_without_seperators, multiplier=3.8): + regions_without_seperators_0 = regions_without_seperators[:, :].sum(axis=0) + + ##plt.plot(regions_without_seperators_0) + ##plt.show() + + sigma_ = 35 # 70#35 + + meda_n_updown = regions_without_seperators_0[len(regions_without_seperators_0) :: -1] + + first_nonzero = next((i for i, x in enumerate(regions_without_seperators_0) if x), 0) + last_nonzero = next((i for i, x in enumerate(meda_n_updown) if x), 0) + + # print(last_nonzero) + # print(isNaN(last_nonzero)) + # last_nonzero=0#halalikh + last_nonzero = len(regions_without_seperators_0) - last_nonzero + + y = regions_without_seperators_0 # [first_nonzero:last_nonzero] + + y_help = np.zeros(len(y) + 20) + + y_help[10 : len(y) + 10] = y + + x = np.array(range(len(y))) + + zneg_rev = -y_help + np.max(y_help) + + zneg = np.zeros(len(zneg_rev) + 20) + + zneg[10 : len(zneg_rev) + 10] = zneg_rev + + z = gaussian_filter1d(y, sigma_) + zneg = gaussian_filter1d(zneg, sigma_) + + peaks_neg, _ = find_peaks(zneg, height=0) + peaks, _ = find_peaks(z, height=0) + + peaks_neg = peaks_neg - 10 - 10 + + last_nonzero = last_nonzero - 100 + first_nonzero = first_nonzero + 200 + + peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & (peaks_neg < last_nonzero)] + + peaks = peaks[(peaks > 0.06 * regions_without_seperators.shape[1]) & (peaks < 0.94 * regions_without_seperators.shape[1])] + peaks_neg = peaks_neg[(peaks_neg > 370) & (peaks_neg < (regions_without_seperators.shape[1] - 370))] + + # print(peaks) + interest_pos = z[peaks] + + interest_pos = interest_pos[interest_pos > 10] + + # plt.plot(z) + # plt.show() + interest_neg = z[peaks_neg] + + min_peaks_pos = np.min(interest_pos) + max_peaks_pos = np.max(interest_pos) + + if max_peaks_pos / min_peaks_pos >= 35: + min_peaks_pos = np.mean(interest_pos) + + min_peaks_neg = 0 # np.min(interest_neg) + + # print(np.min(interest_pos),np.max(interest_pos),np.max(interest_pos)/np.min(interest_pos),'minmax') + # $print(min_peaks_pos) + dis_talaei = (min_peaks_pos - min_peaks_neg) / multiplier + # print(interest_pos) + grenze = min_peaks_pos - dis_talaei # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 + + # print(interest_neg,'interest_neg') + # print(grenze,'grenze') + # print(min_peaks_pos,'min_peaks_pos') + # print(dis_talaei,'dis_talaei') + # print(peaks_neg,'peaks_neg') + + interest_neg_fin = interest_neg[(interest_neg < grenze)] + peaks_neg_fin = peaks_neg[(interest_neg < grenze)] + # interest_neg_fin=interest_neg[(interest_neg p_g_u and peaks_neg_fin[1] > p_g_u) or (peaks_neg_fin[0] < p_g_l and peaks_neg_fin[1] < p_g_l) or ((peaks_neg_fin[0] + 200) < p_m and peaks_neg_fin[1] < p_m) or ((peaks_neg_fin[0] - 200) > p_m and peaks_neg_fin[1] > p_m): + num_col = 1 + peaks_neg_fin = [] + else: + pass + + if num_col == 2: + if (peaks_neg_fin[0] > p_g_u) or (peaks_neg_fin[0] < p_g_l): + num_col = 1 + peaks_neg_fin = [] + else: + pass + + ##print(len(peaks_neg_fin)) + + diff_peaks = np.abs(np.diff(peaks_neg_fin)) + + cut_off = 400 + peaks_neg_true = [] + forest = [] + + # print(len(peaks_neg_fin),'len_') + + for i in range(len(peaks_neg_fin)): + if i == 0: + forest.append(peaks_neg_fin[i]) + if i < (len(peaks_neg_fin) - 1): + if diff_peaks[i] <= cut_off: + forest.append(peaks_neg_fin[i + 1]) + if diff_peaks[i] > cut_off: + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmin(z[forest])]): + peaks_neg_true.append(forest[np.argmin(z[forest])]) + forest = [] + forest.append(peaks_neg_fin[i + 1]) + if i == (len(peaks_neg_fin) - 1): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmin(z[forest])]): + peaks_neg_true.append(forest[np.argmin(z[forest])]) + + num_col = (len(peaks_neg_true)) + 1 + p_l = 0 + p_u = len(y) - 1 + p_m = int(len(y) / 2.0) + p_quarter = int(len(y) / 5.0) + p_g_l = int(len(y) / 4.0) + p_g_u = len(y) - int(len(y) / 4.0) + + p_u_quarter = len(y) - p_quarter + + ##print(num_col,'early') + if num_col == 3: + if (peaks_neg_true[0] > p_g_u and peaks_neg_true[1] > p_g_u) or (peaks_neg_true[0] < p_g_l and peaks_neg_true[1] < p_g_l) or (peaks_neg_true[0] < p_m and (peaks_neg_true[1] + 200) < p_m) or ((peaks_neg_true[0] - 200) > p_m and peaks_neg_true[1] > p_m): + num_col = 1 + peaks_neg_true = [] + elif (peaks_neg_true[0] < p_g_u and peaks_neg_true[0] > p_g_l) and (peaks_neg_true[1] > p_u_quarter): + peaks_neg_true = [peaks_neg_true[0]] + elif (peaks_neg_true[1] < p_g_u and peaks_neg_true[1] > p_g_l) and (peaks_neg_true[0] < p_quarter): + peaks_neg_true = [peaks_neg_true[1]] + else: + pass + + if num_col == 2: + if (peaks_neg_true[0] > p_g_u) or (peaks_neg_true[0] < p_g_l): + num_col = 1 + peaks_neg_true = [] + else: + pass + + diff_peaks_annormal = diff_peaks[diff_peaks < 360] + + if len(diff_peaks_annormal) > 0: + arg_help = np.array(range(len(diff_peaks))) + arg_help_ann = arg_help[diff_peaks < 360] + + peaks_neg_fin_new = [] + + for ii in range(len(peaks_neg_fin)): + if ii in arg_help_ann: + arg_min = np.argmin([interest_neg_fin[ii], interest_neg_fin[ii + 1]]) + if arg_min == 0: + peaks_neg_fin_new.append(peaks_neg_fin[ii]) + else: + peaks_neg_fin_new.append(peaks_neg_fin[ii + 1]) + + elif (ii - 1) in arg_help_ann: + pass + else: + peaks_neg_fin_new.append(peaks_neg_fin[ii]) + else: + peaks_neg_fin_new = peaks_neg_fin + + # plt.plot(gaussian_filter1d(y, sigma_)) + # plt.plot(peaks_neg_true,z[peaks_neg_true],'*') + # plt.plot([0,len(y)], [grenze,grenze]) + # plt.show() + + ##print(len(peaks_neg_true)) + return len(peaks_neg_true), peaks_neg_true + +def find_num_col_only_image(regions_without_seperators, multiplier=3.8): + regions_without_seperators_0 = regions_without_seperators[:, :].sum(axis=0) + + ##plt.plot(regions_without_seperators_0) + ##plt.show() + + sigma_ = 15 + + meda_n_updown = regions_without_seperators_0[len(regions_without_seperators_0) :: -1] + + first_nonzero = next((i for i, x in enumerate(regions_without_seperators_0) if x), 0) + last_nonzero = next((i for i, x in enumerate(meda_n_updown) if x), 0) + + last_nonzero = len(regions_without_seperators_0) - last_nonzero + + y = regions_without_seperators_0 # [first_nonzero:last_nonzero] + + y_help = np.zeros(len(y) + 20) + + y_help[10 : len(y) + 10] = y + + x = np.array(range(len(y))) + + zneg_rev = -y_help + np.max(y_help) + + zneg = np.zeros(len(zneg_rev) + 20) + + zneg[10 : len(zneg_rev) + 10] = zneg_rev + + z = gaussian_filter1d(y, sigma_) + zneg = gaussian_filter1d(zneg, sigma_) + + peaks_neg, _ = find_peaks(zneg, height=0) + peaks, _ = find_peaks(z, height=0) + + peaks_neg = peaks_neg - 10 - 10 + + peaks_neg_org = np.copy(peaks_neg) + + peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & (peaks_neg < last_nonzero)] + + peaks = peaks[(peaks > 0.09 * regions_without_seperators.shape[1]) & (peaks < 0.91 * regions_without_seperators.shape[1])] + + peaks_neg = peaks_neg[(peaks_neg > 500) & (peaks_neg < (regions_without_seperators.shape[1] - 500))] + # print(peaks) + interest_pos = z[peaks] + + interest_pos = interest_pos[interest_pos > 10] + + interest_neg = z[peaks_neg] + min_peaks_pos = np.mean(interest_pos) # np.min(interest_pos) + min_peaks_neg = 0 # np.min(interest_neg) + + # $print(min_peaks_pos) + dis_talaei = (min_peaks_pos - min_peaks_neg) / multiplier + # print(interest_pos) + grenze = min_peaks_pos - dis_talaei # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 + + interest_neg_fin = interest_neg[(interest_neg < grenze)] + peaks_neg_fin = peaks_neg[(interest_neg < grenze)] + + num_col = (len(interest_neg_fin)) + 1 + + p_l = 0 + p_u = len(y) - 1 + p_m = int(len(y) / 2.0) + p_g_l = int(len(y) / 3.0) + p_g_u = len(y) - int(len(y) / 3.0) + + if num_col == 3: + if (peaks_neg_fin[0] > p_g_u and peaks_neg_fin[1] > p_g_u) or (peaks_neg_fin[0] < p_g_l and peaks_neg_fin[1] < p_g_l) or (peaks_neg_fin[0] < p_m and peaks_neg_fin[1] < p_m) or (peaks_neg_fin[0] > p_m and peaks_neg_fin[1] > p_m): + num_col = 1 + else: + pass + + if num_col == 2: + if (peaks_neg_fin[0] > p_g_u) or (peaks_neg_fin[0] < p_g_l): + num_col = 1 + else: + pass + + diff_peaks = np.abs(np.diff(peaks_neg_fin)) + + cut_off = 400 + peaks_neg_true = [] + forest = [] + + for i in range(len(peaks_neg_fin)): + if i == 0: + forest.append(peaks_neg_fin[i]) + if i < (len(peaks_neg_fin) - 1): + if diff_peaks[i] <= cut_off: + forest.append(peaks_neg_fin[i + 1]) + if diff_peaks[i] > cut_off: + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmin(z[forest])]): + peaks_neg_true.append(forest[np.argmin(z[forest])]) + forest = [] + forest.append(peaks_neg_fin[i + 1]) + if i == (len(peaks_neg_fin) - 1): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmin(z[forest])]): + peaks_neg_true.append(forest[np.argmin(z[forest])]) + + num_col = (len(peaks_neg_true)) + 1 + p_l = 0 + p_u = len(y) - 1 + p_m = int(len(y) / 2.0) + p_quarter = int(len(y) / 4.0) + p_g_l = int(len(y) / 3.0) + p_g_u = len(y) - int(len(y) / 3.0) + + p_u_quarter = len(y) - p_quarter + + if num_col == 3: + if (peaks_neg_true[0] > p_g_u and peaks_neg_true[1] > p_g_u) or (peaks_neg_true[0] < p_g_l and peaks_neg_true[1] < p_g_l) or (peaks_neg_true[0] < p_m and peaks_neg_true[1] < p_m) or (peaks_neg_true[0] > p_m and peaks_neg_true[1] > p_m): + num_col = 1 + peaks_neg_true = [] + elif (peaks_neg_true[0] < p_g_u and peaks_neg_true[0] > p_g_l) and (peaks_neg_true[1] > p_u_quarter): + peaks_neg_true = [peaks_neg_true[0]] + elif (peaks_neg_true[1] < p_g_u and peaks_neg_true[1] > p_g_l) and (peaks_neg_true[0] < p_quarter): + peaks_neg_true = [peaks_neg_true[1]] + else: + pass + + if num_col == 2: + if (peaks_neg_true[0] > p_g_u) or (peaks_neg_true[0] < p_g_l): + num_col = 1 + peaks_neg_true = [] + + if num_col == 4: + if len(np.array(peaks_neg_true)[np.array(peaks_neg_true) < p_g_l]) == 2 or len(np.array(peaks_neg_true)[np.array(peaks_neg_true) > (len(y) - p_g_l)]) == 2: + num_col = 1 + peaks_neg_true = [] + else: + pass + + # no deeper hill around found hills + + peaks_fin_true = [] + for i in range(len(peaks_neg_true)): + hill_main = peaks_neg_true[i] + # deep_depth=z[peaks_neg] + hills_around = peaks_neg_org[((peaks_neg_org > hill_main) & (peaks_neg_org <= hill_main + 400)) | ((peaks_neg_org < hill_main) & (peaks_neg_org >= hill_main - 400))] + deep_depth_around = z[hills_around] + + # print(hill_main,z[hill_main],hills_around,deep_depth_around,'manoooo') + try: + if np.min(deep_depth_around) < z[hill_main]: + pass + else: + peaks_fin_true.append(hill_main) + except: + pass + + diff_peaks_annormal = diff_peaks[diff_peaks < 360] + + if len(diff_peaks_annormal) > 0: + arg_help = np.array(range(len(diff_peaks))) + arg_help_ann = arg_help[diff_peaks < 360] + + peaks_neg_fin_new = [] + + for ii in range(len(peaks_neg_fin)): + if ii in arg_help_ann: + arg_min = np.argmin([interest_neg_fin[ii], interest_neg_fin[ii + 1]]) + if arg_min == 0: + peaks_neg_fin_new.append(peaks_neg_fin[ii]) + else: + peaks_neg_fin_new.append(peaks_neg_fin[ii + 1]) + + elif (ii - 1) in arg_help_ann: + pass + else: + peaks_neg_fin_new.append(peaks_neg_fin[ii]) + else: + peaks_neg_fin_new = peaks_neg_fin + + # sometime pages with one columns gives also some negative peaks. delete those peaks + param = z[peaks_neg_true] / float(min_peaks_pos) * 100 + + if len(param[param <= 41]) == 0: + peaks_neg_true = [] + + return len(peaks_fin_true), peaks_fin_true + +def find_num_col_by_vertical_lines(regions_without_seperators, multiplier=3.8): + regions_without_seperators_0 = regions_without_seperators[:, :, 0].sum(axis=0) + + ##plt.plot(regions_without_seperators_0) + ##plt.show() + + sigma_ = 35 # 70#35 + + z = gaussian_filter1d(regions_without_seperators_0, sigma_) + + peaks, _ = find_peaks(z, height=0) + + # print(peaks,'peaksnew') + return peaks + + +def delete_seperator_around(spliter_y, peaks_neg, image_by_region): + # format of subboxes box=[x1, x2 , y1, y2] + + if len(image_by_region.shape) == 3: + for i in range(len(spliter_y) - 1): + for j in range(1, len(peaks_neg[i]) - 1): + image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0] == 6] = 0 + image_by_region[spliter_y[i] : spliter_y[i + 1], peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 1] == 6] = 0 + image_by_region[spliter_y[i] : spliter_y[i + 1], peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 2] == 6] = 0 + + image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0] == 7] = 0 + image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 1] == 7] = 0 + image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 0][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j]), 2] == 7] = 0 + else: + for i in range(len(spliter_y) - 1): + for j in range(1, len(peaks_neg[i]) - 1): + image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j])][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j])] == 6] = 0 + + image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j])][image_by_region[int(spliter_y[i]) : int(spliter_y[i + 1]), peaks_neg[i][j] - int(1.0 / 20.0 * peaks_neg[i][j]) : peaks_neg[i][j] + int(1.0 / 20.0 * peaks_neg[i][j])] == 7] = 0 + return image_by_region + +def return_regions_without_seperators(regions_pre): + kernel = np.ones((5, 5), np.uint8) + regions_without_seperators = ((regions_pre[:, :] != 6) & (regions_pre[:, :] != 0)) * 1 + # regions_without_seperators=( (image_regions_eraly_p[:,:,:]!=6) & (image_regions_eraly_p[:,:,:]!=0) & (image_regions_eraly_p[:,:,:]!=5) & (image_regions_eraly_p[:,:,:]!=8) & (image_regions_eraly_p[:,:,:]!=7))*1 + + regions_without_seperators = regions_without_seperators.astype(np.uint8) + + regions_without_seperators = cv2.erode(regions_without_seperators, kernel, iterations=6) + + return regions_without_seperators + + +def put_drop_out_from_only_drop_model(layout_no_patch, layout1): + + drop_only = (layout_no_patch[:, :, 0] == 4) * 1 + contours_drop, hir_on_drop = return_contours_of_image(drop_only) + contours_drop_parent = return_parent_contours(contours_drop, hir_on_drop) + + areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) + areas_cnt_text = areas_cnt_text / float(drop_only.shape[0] * drop_only.shape[1]) + + contours_drop_parent = [contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz] > 0.00001] + + areas_cnt_text = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > 0.00001] + + contours_drop_parent_final = [] + + for jj in range(len(contours_drop_parent)): + x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) + # boxes.append([int(x), int(y), int(w), int(h)]) + + map_of_drop_contour_bb = np.zeros((layout1.shape[0], layout1.shape[1])) + map_of_drop_contour_bb[y : y + h, x : x + w] = layout1[y : y + h, x : x + w] + + if (((map_of_drop_contour_bb == 1) * 1).sum() / float(((map_of_drop_contour_bb == 5) * 1).sum()) * 100) >= 15: + contours_drop_parent_final.append(contours_drop_parent[jj]) + + layout_no_patch[:, :, 0][layout_no_patch[:, :, 0] == 4] = 0 + + layout_no_patch = cv2.fillPoly(layout_no_patch, pts=contours_drop_parent_final, color=(4, 4, 4)) + + return layout_no_patch + +def putt_bb_of_drop_capitals_of_model_in_patches_in_layout(layout_in_patch): + + drop_only = (layout_in_patch[:, :, 0] == 4) * 1 + contours_drop, hir_on_drop = return_contours_of_image(drop_only) + contours_drop_parent = return_parent_contours(contours_drop, hir_on_drop) + + areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) + areas_cnt_text = areas_cnt_text / float(drop_only.shape[0] * drop_only.shape[1]) + + contours_drop_parent = [contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz] > 0.00001] + + areas_cnt_text = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > 0.001] + + contours_drop_parent_final = [] + + for jj in range(len(contours_drop_parent)): + x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) + layout_in_patch[y : y + h, x : x + w, 0] = 4 + + return layout_in_patch + +def check_any_text_region_in_model_one_is_main_or_header(regions_model_1,regions_model_full,contours_only_text_parent,all_box_coord,all_found_texline_polygons,slopes,contours_only_text_parent_d_ordered): + #text_only=(regions_model_1[:,:]==1)*1 + #contours_only_text,hir_on_text=self.return_contours_of_image(text_only) + + """ + contours_only_text_parent=self.return_parent_contours( contours_only_text,hir_on_text) + + areas_cnt_text=np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) + areas_cnt_text=areas_cnt_text/float(text_only.shape[0]*text_only.shape[1]) + + ###areas_cnt_text_h=np.array([cv2.contourArea(contours_only_text_parent_h[j]) for j in range(len(contours_only_text_parent_h))]) + ###areas_cnt_text_h=areas_cnt_text_h/float(text_only_h.shape[0]*text_only_h.shape[1]) + + ###contours_only_text_parent=[contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz]>0.0002] + contours_only_text_parent=[contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz]>0.00001] + """ + + cx_main,cy_main ,x_min_main , x_max_main, y_min_main ,y_max_main,y_corr_x_min_from_argmin=find_new_features_of_contoures(contours_only_text_parent) + + length_con=x_max_main-x_min_main + height_con=y_max_main-y_min_main + + + + all_found_texline_polygons_main=[] + all_found_texline_polygons_head=[] + + all_box_coord_main=[] + all_box_coord_head=[] + + slopes_main=[] + slopes_head=[] + + contours_only_text_parent_main=[] + contours_only_text_parent_head=[] + + contours_only_text_parent_main_d=[] + contours_only_text_parent_head_d=[] + + for ii in range(len(contours_only_text_parent)): + con=contours_only_text_parent[ii] + img=np.zeros((regions_model_1.shape[0],regions_model_1.shape[1],3)) + img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) + + + + all_pixels=((img[:,:,0]==255)*1).sum() + + pixels_header=( ( (img[:,:,0]==255) & (regions_model_full[:,:,0]==2) )*1 ).sum() + pixels_main=all_pixels-pixels_header + + + if (pixels_header>=pixels_main) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 + contours_only_text_parent_head.append(con) + if contours_only_text_parent_d_ordered is not None: + contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) + all_box_coord_head.append(all_box_coord[ii]) + slopes_head.append(slopes[ii]) + all_found_texline_polygons_head.append(all_found_texline_polygons[ii]) + else: + regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 + contours_only_text_parent_main.append(con) + if contours_only_text_parent_d_ordered is not None: + contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) + all_box_coord_main.append(all_box_coord[ii]) + slopes_main.append(slopes[ii]) + all_found_texline_polygons_main.append(all_found_texline_polygons[ii]) + + #print(all_pixels,pixels_main,pixels_header) + + + + #plt.imshow(img[:,:,0]) + #plt.show() + return regions_model_1,contours_only_text_parent_main,contours_only_text_parent_head,all_box_coord_main,all_box_coord_head,all_found_texline_polygons_main,all_found_texline_polygons_head,slopes_main,slopes_head,contours_only_text_parent_main_d,contours_only_text_parent_head_d + +def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col): + # print(textlines_con) + # textlines_con=textlines_con.astype(np.uint32) + + textlines_con_changed = [] + for m1 in range(len(textlines_con)): + + # textlines_tot=textlines_con[m1] + # textlines_tot=textlines_tot.astype() + textlines_tot = [] + textlines_tot_org_form = [] + # print(textlines_tot) + + for nn in range(len(textlines_con[m1])): + textlines_tot.append(np.array(textlines_con[m1][nn], dtype=np.int32)) + textlines_tot_org_form.append(textlines_con[m1][nn]) + + ##img_text_all=np.zeros((textline_iamge.shape[0],textline_iamge.shape[1])) + ##img_text_all=cv2.fillPoly(img_text_all, pts =textlines_tot , color=(1,1,1)) + + ##plt.imshow(img_text_all) + ##plt.show() + areas_cnt_text = np.array([cv2.contourArea(textlines_tot[j]) for j in range(len(textlines_tot))]) + areas_cnt_text = areas_cnt_text / float(textline_iamge.shape[0] * textline_iamge.shape[1]) + indexes_textlines = np.array(range(len(textlines_tot))) + + # print(areas_cnt_text,np.min(areas_cnt_text),np.max(areas_cnt_text)) + if num_col == 0: + min_area = 0.0004 + elif num_col == 1: + min_area = 0.0003 + else: + min_area = 0.0001 + indexes_textlines_small = indexes_textlines[areas_cnt_text < min_area] + + # print(indexes_textlines) + + textlines_small = [] + textlines_small_org_form = [] + for i in indexes_textlines_small: + textlines_small.append(textlines_tot[i]) + textlines_small_org_form.append(textlines_tot_org_form[i]) + + textlines_big = [] + textlines_big_org_form = [] + for i in list(set(indexes_textlines) - set(indexes_textlines_small)): + textlines_big.append(textlines_tot[i]) + textlines_big_org_form.append(textlines_tot_org_form[i]) + + img_textline_s = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) + img_textline_s = cv2.fillPoly(img_textline_s, pts=textlines_small, color=(1, 1, 1)) + + img_textline_b = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) + img_textline_b = cv2.fillPoly(img_textline_b, pts=textlines_big, color=(1, 1, 1)) + + sum_small_big_all = img_textline_s + img_textline_b + sum_small_big_all2 = (sum_small_big_all[:, :] == 2) * 1 + + sum_intersection_sb = sum_small_big_all2.sum(axis=1).sum() + + if sum_intersection_sb > 0: + + dis_small_from_bigs_tot = [] + for z1 in range(len(textlines_small)): + # print(len(textlines_small),'small') + intersections = [] + for z2 in range(len(textlines_big)): + img_text = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) + img_text = cv2.fillPoly(img_text, pts=[textlines_small[z1]], color=(1, 1, 1)) + + img_text2 = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1])) + img_text2 = cv2.fillPoly(img_text2, pts=[textlines_big[z2]], color=(1, 1, 1)) + + sum_small_big = img_text2 + img_text + sum_small_big_2 = (sum_small_big[:, :] == 2) * 1 + + sum_intersection = sum_small_big_2.sum(axis=1).sum() + + # print(sum_intersection) + + intersections.append(sum_intersection) + + if len(np.array(intersections)[np.array(intersections) > 0]) == 0: + intersections = [] + + try: + dis_small_from_bigs_tot.append(np.argmax(intersections)) + except: + dis_small_from_bigs_tot.append(-1) + + smalls_list = np.array(dis_small_from_bigs_tot)[np.array(dis_small_from_bigs_tot) >= 0] + + # index_small_textlines_rest=list( set(indexes_textlines_small)-set(smalls_list) ) + + textlines_big_with_change = [] + textlines_big_with_change_con = [] + textlines_small_with_change = [] + + for z in list(set(smalls_list)): + index_small_textlines = list(np.where(np.array(dis_small_from_bigs_tot) == z)[0]) + # print(z,index_small_textlines) + + img_text2 = np.zeros((textline_iamge.shape[0], textline_iamge.shape[1], 3)) + img_text2 = cv2.fillPoly(img_text2, pts=[textlines_big[z]], color=(255, 255, 255)) + + textlines_big_with_change.append(z) + + for k in index_small_textlines: + img_text2 = cv2.fillPoly(img_text2, pts=[textlines_small[k]], color=(255, 255, 255)) + textlines_small_with_change.append(k) + + img_text2 = img_text2.astype(np.uint8) + imgray = cv2.cvtColor(img_text2, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + cont, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(cont[0],type(cont)) + + textlines_big_with_change_con.append(cont) + textlines_big_org_form[z] = cont[0] + + # plt.imshow(img_text2) + # plt.show() + + # print(textlines_big_with_change,'textlines_big_with_change') + # print(textlines_small_with_change,'textlines_small_with_change') + # print(textlines_big) + textlines_con_changed.append(textlines_big_org_form) + + else: + textlines_con_changed.append(textlines_big_org_form) + return textlines_con_changed + +def order_and_id_of_texts(found_polygons_text_region, found_polygons_text_region_h, matrix_of_orders, indexes_sorted, index_of_types, kind_of_texts, ref_point): + indexes_sorted = np.array(indexes_sorted) + index_of_types = np.array(index_of_types) + kind_of_texts = np.array(kind_of_texts) + + id_of_texts = [] + order_of_texts = [] + + index_of_types_1 = index_of_types[kind_of_texts == 1] + indexes_sorted_1 = indexes_sorted[kind_of_texts == 1] + + index_of_types_2 = index_of_types[kind_of_texts == 2] + indexes_sorted_2 = indexes_sorted[kind_of_texts == 2] + + ##print(index_of_types,'index_of_types') + ##print(kind_of_texts,'kind_of_texts') + ##print(len(found_polygons_text_region),'found_polygons_text_region') + ##print(index_of_types_1,'index_of_types_1') + ##print(indexes_sorted_1,'indexes_sorted_1') + index_b = 0 + ref_point + for mm in range(len(found_polygons_text_region)): + + id_of_texts.append("r" + str(index_b)) + interest = indexes_sorted_1[indexes_sorted_1 == index_of_types_1[mm]] + + if len(interest) > 0: + order_of_texts.append(interest[0]) + index_b += 1 + else: + pass + + for mm in range(len(found_polygons_text_region_h)): + id_of_texts.append("r" + str(index_b)) + interest = indexes_sorted_2[index_of_types_2[mm]] + order_of_texts.append(interest) + index_b += 1 + + return order_of_texts, id_of_texts + +def order_of_regions(textline_mask, contours_main, contours_header, y_ref): + + ##plt.imshow(textline_mask) + ##plt.show() + """ + print(len(contours_main),'contours_main') + mada_n=textline_mask.sum(axis=1) + y=mada_n[:] + + y_help=np.zeros(len(y)+40) + y_help[20:len(y)+20]=y + x=np.array( range(len(y)) ) + + + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + + ##plt.imshow(textline_mask[:,:]) + ##plt.show() + + + sigma_gaus=8 + + z= gaussian_filter1d(y_help, sigma_gaus) + zneg_rev=-y_help+np.max(y_help) + + zneg=np.zeros(len(zneg_rev)+40) + zneg[20:len(zneg_rev)+20]=zneg_rev + zneg= gaussian_filter1d(zneg, sigma_gaus) + + + peaks, _ = find_peaks(z, height=0) + peaks_neg, _ = find_peaks(zneg, height=0) + + peaks_neg=peaks_neg-20-20 + peaks=peaks-20 + """ + + textline_sum_along_width = textline_mask.sum(axis=1) + + y = textline_sum_along_width[:] + y_padded = np.zeros(len(y) + 40) + y_padded[20 : len(y) + 20] = y + x = np.array(range(len(y))) + + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + + sigma_gaus = 8 + + z = gaussian_filter1d(y_padded, sigma_gaus) + zneg_rev = -y_padded + np.max(y_padded) + + zneg = np.zeros(len(zneg_rev) + 40) + zneg[20 : len(zneg_rev) + 20] = zneg_rev + zneg = gaussian_filter1d(zneg, sigma_gaus) + + peaks, _ = find_peaks(z, height=0) + peaks_neg, _ = find_peaks(zneg, height=0) + + peaks_neg = peaks_neg - 20 - 20 + peaks = peaks - 20 + + ##plt.plot(z) + ##plt.show() + + if contours_main != None: + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + + if len(contours_header) != None: + areas_header = np.array([cv2.contourArea(contours_header[j]) for j in range(len(contours_header))]) + M_header = [cv2.moments(contours_header[j]) for j in range(len(contours_header))] + cx_header = [(M_header[j]["m10"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] + cy_header = [(M_header[j]["m01"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] + + x_min_header = np.array([np.min(contours_header[j][:, 0, 0]) for j in range(len(contours_header))]) + x_max_header = np.array([np.max(contours_header[j][:, 0, 0]) for j in range(len(contours_header))]) + + y_min_header = np.array([np.min(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) + y_max_header = np.array([np.max(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) + # print(cy_main,'mainy') + + peaks_neg_new = [] + + peaks_neg_new.append(0 + y_ref) + for iii in range(len(peaks_neg)): + peaks_neg_new.append(peaks_neg[iii] + y_ref) + + peaks_neg_new.append(textline_mask.shape[0] + y_ref) + + if len(cy_main) > 0 and np.max(cy_main) > np.max(peaks_neg_new): + cy_main = np.array(cy_main) * (np.max(peaks_neg_new) / np.max(cy_main)) - 10 + + if contours_main != None: + indexer_main = np.array(range(len(contours_main))) + + if contours_main != None: + len_main = len(contours_main) + else: + len_main = 0 + + matrix_of_orders = np.zeros((len(contours_main) + len(contours_header), 5)) + + matrix_of_orders[:, 0] = np.array(range(len(contours_main) + len(contours_header))) + + matrix_of_orders[: len(contours_main), 1] = 1 + matrix_of_orders[len(contours_main) :, 1] = 2 + + matrix_of_orders[: len(contours_main), 2] = cx_main + matrix_of_orders[len(contours_main) :, 2] = cx_header + + matrix_of_orders[: len(contours_main), 3] = cy_main + matrix_of_orders[len(contours_main) :, 3] = cy_header + + matrix_of_orders[: len(contours_main), 4] = np.array(range(len(contours_main))) + matrix_of_orders[len(contours_main) :, 4] = np.array(range(len(contours_header))) + + # print(peaks_neg_new,'peaks_neg_new') + + # print(matrix_of_orders,'matrix_of_orders') + # print(peaks_neg_new,np.max(peaks_neg_new)) + final_indexers_sorted = [] + final_types = [] + final_index_type = [] + for i in range(len(peaks_neg_new) - 1): + top = peaks_neg_new[i] + down = peaks_neg_new[i + 1] + + # print(top,down,'topdown') + + indexes_in = matrix_of_orders[:, 0][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + cxs_in = matrix_of_orders[:, 2][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + cys_in = matrix_of_orders[:, 3][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + types_of_text = matrix_of_orders[:, 1][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + index_types_of_text = matrix_of_orders[:, 4][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + + # print(top,down) + # print(cys_in,'cyyyins') + # print(indexes_in,'indexes') + sorted_inside = np.argsort(cxs_in) + + ind_in_int = indexes_in[sorted_inside] + ind_in_type = types_of_text[sorted_inside] + ind_ind_type = index_types_of_text[sorted_inside] + + for j in range(len(ind_in_int)): + final_indexers_sorted.append(int(ind_in_int[j])) + final_types.append(int(ind_in_type[j])) + final_index_type.append(int(ind_ind_type[j])) + + ##matrix_of_orders[:len_main,4]=final_indexers_sorted[:] + + # print(peaks_neg_new,'peaks') + # print(final_indexers_sorted,'indexsorted') + # print(final_types,'types') + # print(final_index_type,'final_index_type') + + return final_indexers_sorted, matrix_of_orders, final_types, final_index_type + +def implent_law_head_main_not_parallel(text_regions): + # print(text_regions.shape) + text_indexes = [1, 2] # 1: main text , 2: header , 3: comments + + for t_i in text_indexes: + textline_mask = text_regions[:, :] == t_i + textline_mask = textline_mask * 255.0 + + textline_mask = textline_mask.astype(np.uint8) + textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) + kernel = np.ones((5, 5), np.uint8) + + # print(type(textline_mask),np.unique(textline_mask),textline_mask.shape) + imgray = cv2.cvtColor(textline_mask, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + if t_i == 1: + contours_main, hirarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + # print(type(contours_main)) + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + # print(contours_main[0],np.shape(contours_main[0]),contours_main[0][:,0,0]) + elif t_i == 2: + contours_header, hirarchy = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + # print(type(contours_header)) + areas_header = np.array([cv2.contourArea(contours_header[j]) for j in range(len(contours_header))]) + M_header = [cv2.moments(contours_header[j]) for j in range(len(contours_header))] + cx_header = [(M_header[j]["m10"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] + cy_header = [(M_header[j]["m01"] / (M_header[j]["m00"] + 1e-32)) for j in range(len(M_header))] + + x_min_header = np.array([np.min(contours_header[j][:, 0, 0]) for j in range(len(contours_header))]) + x_max_header = np.array([np.max(contours_header[j][:, 0, 0]) for j in range(len(contours_header))]) + + y_min_header = np.array([np.min(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) + y_max_header = np.array([np.max(contours_header[j][:, 0, 1]) for j in range(len(contours_header))]) + + args = np.array(range(1, len(cy_header) + 1)) + args_main = np.array(range(1, len(cy_main) + 1)) + for jj in range(len(contours_main)): + headers_in_main = [(cy_header > y_min_main[jj]) & ((cy_header < y_max_main[jj]))] + mains_in_main = [(cy_main > y_min_main[jj]) & ((cy_main < y_max_main[jj]))] + args_log = args * headers_in_main + res = args_log[args_log > 0] + res_true = res - 1 + + args_log_main = args_main * mains_in_main + res_main = args_log_main[args_log_main > 0] + res_true_main = res_main - 1 + + if len(res_true) > 0: + sum_header = np.sum(areas_header[res_true]) + sum_main = np.sum(areas_main[res_true_main]) + if sum_main > sum_header: + cnt_int = [contours_header[j] for j in res_true] + text_regions = cv2.fillPoly(text_regions, pts=cnt_int, color=(1, 1, 1)) + else: + cnt_int = [contours_main[j] for j in res_true_main] + text_regions = cv2.fillPoly(text_regions, pts=cnt_int, color=(2, 2, 2)) + + for jj in range(len(contours_header)): + main_in_header = [(cy_main > y_min_header[jj]) & ((cy_main < y_max_header[jj]))] + header_in_header = [(cy_header > y_min_header[jj]) & ((cy_header < y_max_header[jj]))] + args_log = args_main * main_in_header + res = args_log[args_log > 0] + res_true = res - 1 + + args_log_header = args * header_in_header + res_header = args_log_header[args_log_header > 0] + res_true_header = res_header - 1 + + if len(res_true) > 0: + + sum_header = np.sum(areas_header[res_true_header]) + sum_main = np.sum(areas_main[res_true]) + + if sum_main > sum_header: + + cnt_int = [contours_header[j] for j in res_true_header] + text_regions = cv2.fillPoly(text_regions, pts=cnt_int, color=(1, 1, 1)) + else: + cnt_int = [contours_main[j] for j in res_true] + text_regions = cv2.fillPoly(text_regions, pts=cnt_int, color=(2, 2, 2)) + + return text_regions + + +def return_hor_spliter_by_index(peaks_neg_fin_t, x_min_hor_some, x_max_hor_some): + + arg_min_hor_sort = np.argsort(x_min_hor_some) + x_min_hor_some_sort = np.sort(x_min_hor_some) + x_max_hor_some_sort = x_max_hor_some[arg_min_hor_sort] + + arg_minmax = np.array(range(len(peaks_neg_fin_t))) + indexer_lines = [] + indexes_to_delete = [] + indexer_lines_deletions_len = [] + indexr_uniq_ind = [] + for i in range(len(x_min_hor_some_sort)): + min_h = peaks_neg_fin_t - x_min_hor_some_sort[i] + max_h = peaks_neg_fin_t - x_max_hor_some_sort[i] + + min_h[0] = min_h[0] # +20 + max_h[len(max_h) - 1] = max_h[len(max_h) - 1] ##-20 + + min_h_neg = arg_minmax[(min_h < 0) & (np.abs(min_h) < 360)] + max_h_neg = arg_minmax[(max_h >= 0) & (np.abs(max_h) < 360)] + + if len(min_h_neg) > 0 and len(max_h_neg) > 0: + deletions = list(range(min_h_neg[0] + 1, max_h_neg[0])) + unique_delets_int = [] + # print(deletions,len(deletions),'delii') + if len(deletions) > 0: + # print(deletions,len(deletions),'delii2') + + for j in range(len(deletions)): + indexes_to_delete.append(deletions[j]) + # print(deletions,indexes_to_delete,'badiii') + unique_delets = np.unique(indexes_to_delete) + # print(min_h_neg[0],unique_delets) + unique_delets_int = unique_delets[unique_delets < min_h_neg[0]] + + indexer_lines_deletions_len.append(len(deletions)) + indexr_uniq_ind.append([deletions]) + + else: + indexer_lines_deletions_len.append(0) + indexr_uniq_ind.append(-999) + + index_line_true = min_h_neg[0] - len(unique_delets_int) + # print(index_line_true) + if index_line_true > 0 and min_h_neg[0] >= 2: + index_line_true = index_line_true + else: + index_line_true = min_h_neg[0] + + indexer_lines.append(index_line_true) + + if len(unique_delets_int) > 0: + for dd in range(len(unique_delets_int)): + indexes_to_delete.append(unique_delets_int[dd]) + else: + indexer_lines.append(-999) + indexer_lines_deletions_len.append(-999) + indexr_uniq_ind.append(-999) + + peaks_true = [] + for m in range(len(peaks_neg_fin_t)): + if m in indexes_to_delete: + pass + else: + peaks_true.append(peaks_neg_fin_t[m]) + return indexer_lines, peaks_true, arg_min_hor_sort, indexer_lines_deletions_len, indexr_uniq_ind + +def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new(img_p_in_ver, img_in_hor): + + # plt.imshow(img_in_hor) + # plt.show() + + # img_p_in_ver = cv2.erode(img_p_in_ver, self.kernel, iterations=2) + img_p_in_ver = img_p_in_ver.astype(np.uint8) + img_p_in_ver = np.repeat(img_p_in_ver[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(img_p_in_ver, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_lines_ver, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + slope_lines_ver, dist_x_ver, x_min_main_ver, x_max_main_ver, cy_main_ver, slope_lines_org_ver, y_min_main_ver, y_max_main_ver, cx_main_ver = find_features_of_lines(contours_lines_ver) + + for i in range(len(x_min_main_ver)): + img_p_in_ver[int(y_min_main_ver[i]) : int(y_min_main_ver[i]) + 30, int(cx_main_ver[i]) - 25 : int(cx_main_ver[i]) + 25, 0] = 0 + img_p_in_ver[int(y_max_main_ver[i]) - 30 : int(y_max_main_ver[i]), int(cx_main_ver[i]) - 25 : int(cx_main_ver[i]) + 25, 0] = 0 + + # plt.imshow(img_p_in_ver[:,:,0]) + # plt.show() + img_in_hor = img_in_hor.astype(np.uint8) + img_in_hor = np.repeat(img_in_hor[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(img_in_hor, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_lines_hor, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + slope_lines_hor, dist_x_hor, x_min_main_hor, x_max_main_hor, cy_main_hor, slope_lines_org_hor, y_min_main_hor, y_max_main_hor, cx_main_hor = find_features_of_lines(contours_lines_hor) + + args_hor = np.array(range(len(slope_lines_hor))) + all_args_uniq = contours_in_same_horizon(cy_main_hor) + # print(all_args_uniq,'all_args_uniq') + if len(all_args_uniq) > 0: + if type(all_args_uniq[0]) is list: + special_seperators = [] + contours_new = [] + for dd in range(len(all_args_uniq)): + merged_all = None + some_args = args_hor[all_args_uniq[dd]] + some_cy = cy_main_hor[all_args_uniq[dd]] + some_x_min = x_min_main_hor[all_args_uniq[dd]] + some_x_max = x_max_main_hor[all_args_uniq[dd]] + + # img_in=np.zeros(seperators_closeup_n[:,:,2].shape) + for jv in range(len(some_args)): + + img_p_in = cv2.fillPoly(img_in_hor, pts=[contours_lines_hor[some_args[jv]]], color=(1, 1, 1)) + img_p_in[int(np.mean(some_cy)) - 5 : int(np.mean(some_cy)) + 5, int(np.min(some_x_min)) : int(np.max(some_x_max))] = 1 + + sum_dis = dist_x_hor[some_args].sum() + diff_max_min_uniques = np.max(x_max_main_hor[some_args]) - np.min(x_min_main_hor[some_args]) + + # print( sum_dis/float(diff_max_min_uniques) ,diff_max_min_uniques/float(img_p_in_ver.shape[1]),dist_x_hor[some_args].sum(),diff_max_min_uniques,np.mean( dist_x_hor[some_args]),np.std( dist_x_hor[some_args]) ) + + if diff_max_min_uniques > sum_dis and ((sum_dis / float(diff_max_min_uniques)) > 0.85) and ((diff_max_min_uniques / float(img_p_in_ver.shape[1])) > 0.85) and np.std(dist_x_hor[some_args]) < (0.55 * np.mean(dist_x_hor[some_args])): + # print(dist_x_hor[some_args],dist_x_hor[some_args].sum(),np.min(x_min_main_hor[some_args]) ,np.max(x_max_main_hor[some_args]),'jalibdi') + # print(np.mean( dist_x_hor[some_args] ),np.std( dist_x_hor[some_args] ),np.var( dist_x_hor[some_args] ),'jalibdiha') + special_seperators.append(np.mean(cy_main_hor[some_args])) + + else: + img_p_in = img_in_hor + special_seperators = [] + else: + img_p_in = img_in_hor + special_seperators = [] + + img_p_in_ver[:, :, 0][img_p_in_ver[:, :, 0] == 255] = 1 + # print(img_p_in_ver.shape,np.unique(img_p_in_ver[:,:,0])) + + # plt.imshow(img_p_in[:,:,0]) + # plt.show() + + # plt.imshow(img_p_in_ver[:,:,0]) + # plt.show() + sep_ver_hor = img_p_in + img_p_in_ver + # print(sep_ver_hor.shape,np.unique(sep_ver_hor[:,:,0]),'sep_ver_horsep_ver_horsep_ver_hor') + # plt.imshow(sep_ver_hor[:,:,0]) + # plt.show() + + sep_ver_hor_cross = (sep_ver_hor[:, :, 0] == 2) * 1 + + sep_ver_hor_cross = np.repeat(sep_ver_hor_cross[:, :, np.newaxis], 3, axis=2) + sep_ver_hor_cross = sep_ver_hor_cross.astype(np.uint8) + imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_cross, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + cx_cross, cy_cross, _, _, _, _, _ = find_new_features_of_contoures(contours_cross) + + for ii in range(len(cx_cross)): + img_p_in[int(cy_cross[ii]) - 30 : int(cy_cross[ii]) + 30, int(cx_cross[ii]) + 5 : int(cx_cross[ii]) + 40, 0] = 0 + img_p_in[int(cy_cross[ii]) - 30 : int(cy_cross[ii]) + 30, int(cx_cross[ii]) - 40 : int(cx_cross[ii]) - 4, 0] = 0 + + # plt.imshow(img_p_in[:,:,0]) + # plt.show() + + return img_p_in[:, :, 0], special_seperators + +def return_points_with_boundies(peaks_neg_fin, first_point, last_point): + peaks_neg_tot = [] + peaks_neg_tot.append(first_point) + for ii in range(len(peaks_neg_fin)): + peaks_neg_tot.append(peaks_neg_fin[ii]) + peaks_neg_tot.append(last_point) + return peaks_neg_tot + +def find_number_of_columns_in_document(region_pre_p, num_col_classifier, pixel_lines, contours_h=None): + + seperators_closeup = ((region_pre_p[:, :, :] == pixel_lines)) * 1 + + seperators_closeup[0:110, :, :] = 0 + seperators_closeup[seperators_closeup.shape[0] - 150 :, :, :] = 0 + + kernel = np.ones((5, 5), np.uint8) + + seperators_closeup = seperators_closeup.astype(np.uint8) + seperators_closeup = cv2.dilate(seperators_closeup, kernel, iterations=1) + seperators_closeup = cv2.erode(seperators_closeup, kernel, iterations=1) + + ##plt.imshow(seperators_closeup[:,:,0]) + ##plt.show() + seperators_closeup_new = np.zeros((seperators_closeup.shape[0], seperators_closeup.shape[1])) + + ##_,seperators_closeup_n=self.combine_hor_lines_and_delete_cross_points_and_get_lines_features_back(region_pre_p[:,:,0]) + seperators_closeup_n = np.copy(seperators_closeup) + + seperators_closeup_n = seperators_closeup_n.astype(np.uint8) + ##plt.imshow(seperators_closeup_n[:,:,0]) + ##plt.show() + + seperators_closeup_n_binary = np.zeros((seperators_closeup_n.shape[0], seperators_closeup_n.shape[1])) + seperators_closeup_n_binary[:, :] = seperators_closeup_n[:, :, 0] + + seperators_closeup_n_binary[:, :][seperators_closeup_n_binary[:, :] != 0] = 1 + # seperators_closeup_n_binary[:,:][seperators_closeup_n_binary[:,:]==0]=255 + # seperators_closeup_n_binary[:,:][seperators_closeup_n_binary[:,:]==-255]=0 + + # seperators_closeup_n_binary=(seperators_closeup_n_binary[:,:]==2)*1 + + # gray = cv2.cvtColor(seperators_closeup_n, cv2.COLOR_BGR2GRAY) + + # print(np.unique(seperators_closeup_n_binary)) + + ##plt.imshow(seperators_closeup_n_binary) + ##plt.show() + + # print( np.unique(gray),np.unique(seperators_closeup_n[:,:,1]) ) + + gray = cv2.bitwise_not(seperators_closeup_n_binary) + gray = gray.astype(np.uint8) + + ##plt.imshow(gray) + ##plt.show() + bw = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 15, -2) + ##plt.imshow(bw[:,:]) + ##plt.show() + + horizontal = np.copy(bw) + vertical = np.copy(bw) + + cols = horizontal.shape[1] + horizontal_size = cols // 30 + # Create structure element for extracting horizontal lines through morphology operations + horizontalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (horizontal_size, 1)) + # Apply morphology operations + horizontal = cv2.erode(horizontal, horizontalStructure) + horizontal = cv2.dilate(horizontal, horizontalStructure) + + kernel = np.ones((5, 5), np.uint8) + + horizontal = cv2.dilate(horizontal, kernel, iterations=2) + horizontal = cv2.erode(horizontal, kernel, iterations=2) + # plt.imshow(horizontal) + # plt.show() + + rows = vertical.shape[0] + verticalsize = rows // 30 + # Create structure element for extracting vertical lines through morphology operations + verticalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (1, verticalsize)) + # Apply morphology operations + vertical = cv2.erode(vertical, verticalStructure) + vertical = cv2.dilate(vertical, verticalStructure) + + vertical = cv2.dilate(vertical, kernel, iterations=1) + # Show extracted vertical lines + + horizontal, special_seperators = combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new(vertical, horizontal) + + ##plt.imshow(vertical) + ##plt.show() + # print(vertical.shape,np.unique(vertical),'verticalvertical') + seperators_closeup_new[:, :][vertical[:, :] != 0] = 1 + seperators_closeup_new[:, :][horizontal[:, :] != 0] = 1 + + ##plt.imshow(seperators_closeup_new) + ##plt.show() + ##seperators_closeup_n + vertical = np.repeat(vertical[:, :, np.newaxis], 3, axis=2) + vertical = vertical.astype(np.uint8) + + ##plt.plot(vertical[:,:,0].sum(axis=0)) + ##plt.show() + + # plt.plot(vertical[:,:,0].sum(axis=1)) + # plt.show() + + imgray = cv2.cvtColor(vertical, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_line_vers, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = find_features_of_lines(contours_line_vers) + # print(slope_lines,'vertical') + args = np.array(range(len(slope_lines))) + args_ver = args[slope_lines == 1] + dist_x_ver = dist_x[slope_lines == 1] + y_min_main_ver = y_min_main[slope_lines == 1] + y_max_main_ver = y_max_main[slope_lines == 1] + x_min_main_ver = x_min_main[slope_lines == 1] + x_max_main_ver = x_max_main[slope_lines == 1] + cx_main_ver = cx_main[slope_lines == 1] + dist_y_ver = y_max_main_ver - y_min_main_ver + len_y = seperators_closeup.shape[0] / 3.0 + + # plt.imshow(horizontal) + # plt.show() + + horizontal = np.repeat(horizontal[:, :, np.newaxis], 3, axis=2) + horizontal = horizontal.astype(np.uint8) + imgray = cv2.cvtColor(horizontal, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_line_hors, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = find_features_of_lines(contours_line_hors) + + slope_lines_org_hor = slope_lines_org[slope_lines == 0] + args = np.array(range(len(slope_lines))) + len_x = seperators_closeup.shape[1] / 5.0 + + dist_y = np.abs(y_max_main - y_min_main) + + args_hor = args[slope_lines == 0] + dist_x_hor = dist_x[slope_lines == 0] + y_min_main_hor = y_min_main[slope_lines == 0] + y_max_main_hor = y_max_main[slope_lines == 0] + x_min_main_hor = x_min_main[slope_lines == 0] + x_max_main_hor = x_max_main[slope_lines == 0] + dist_y_hor = dist_y[slope_lines == 0] + cy_main_hor = cy_main[slope_lines == 0] + + args_hor = args_hor[dist_x_hor >= len_x / 2.0] + x_max_main_hor = x_max_main_hor[dist_x_hor >= len_x / 2.0] + x_min_main_hor = x_min_main_hor[dist_x_hor >= len_x / 2.0] + cy_main_hor = cy_main_hor[dist_x_hor >= len_x / 2.0] + y_min_main_hor = y_min_main_hor[dist_x_hor >= len_x / 2.0] + y_max_main_hor = y_max_main_hor[dist_x_hor >= len_x / 2.0] + dist_y_hor = dist_y_hor[dist_x_hor >= len_x / 2.0] + + slope_lines_org_hor = slope_lines_org_hor[dist_x_hor >= len_x / 2.0] + dist_x_hor = dist_x_hor[dist_x_hor >= len_x / 2.0] + + matrix_of_lines_ch = np.zeros((len(cy_main_hor) + len(cx_main_ver), 10)) + + matrix_of_lines_ch[: len(cy_main_hor), 0] = args_hor + matrix_of_lines_ch[len(cy_main_hor) :, 0] = args_ver + + matrix_of_lines_ch[len(cy_main_hor) :, 1] = cx_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 2] = x_min_main_hor + 50 # x_min_main_hor+150 + matrix_of_lines_ch[len(cy_main_hor) :, 2] = x_min_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 3] = x_max_main_hor - 50 # x_max_main_hor-150 + matrix_of_lines_ch[len(cy_main_hor) :, 3] = x_max_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 4] = dist_x_hor + matrix_of_lines_ch[len(cy_main_hor) :, 4] = dist_x_ver + + matrix_of_lines_ch[: len(cy_main_hor), 5] = cy_main_hor + + matrix_of_lines_ch[: len(cy_main_hor), 6] = y_min_main_hor + matrix_of_lines_ch[len(cy_main_hor) :, 6] = y_min_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 7] = y_max_main_hor + matrix_of_lines_ch[len(cy_main_hor) :, 7] = y_max_main_ver + + matrix_of_lines_ch[: len(cy_main_hor), 8] = dist_y_hor + matrix_of_lines_ch[len(cy_main_hor) :, 8] = dist_y_ver + + matrix_of_lines_ch[len(cy_main_hor) :, 9] = 1 + + if contours_h is not None: + slope_lines_head, dist_x_head, x_min_main_head, x_max_main_head, cy_main_head, slope_lines_org_head, y_min_main_head, y_max_main_head, cx_main_head = find_features_of_lines(contours_h) + matrix_l_n = np.zeros((matrix_of_lines_ch.shape[0] + len(cy_main_head), matrix_of_lines_ch.shape[1])) + matrix_l_n[: matrix_of_lines_ch.shape[0], :] = np.copy(matrix_of_lines_ch[:, :]) + args_head = np.array(range(len(cy_main_head))) + len(cy_main_hor) + + matrix_l_n[matrix_of_lines_ch.shape[0] :, 0] = args_head + matrix_l_n[matrix_of_lines_ch.shape[0] :, 2] = x_min_main_head + 30 + matrix_l_n[matrix_of_lines_ch.shape[0] :, 3] = x_max_main_head - 30 + + matrix_l_n[matrix_of_lines_ch.shape[0] :, 4] = dist_x_head + + matrix_l_n[matrix_of_lines_ch.shape[0] :, 5] = y_min_main_head - 3 - 8 + matrix_l_n[matrix_of_lines_ch.shape[0] :, 6] = y_min_main_head - 5 - 8 + matrix_l_n[matrix_of_lines_ch.shape[0] :, 7] = y_min_main_head + 1 - 8 + matrix_l_n[matrix_of_lines_ch.shape[0] :, 8] = 4 + + matrix_of_lines_ch = np.copy(matrix_l_n) + + # print(matrix_of_lines_ch) + + """ + + + + seperators_closeup=seperators_closeup.astype(np.uint8) + imgray = cv2.cvtColor(seperators_closeup, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_lines,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + + slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=find_features_of_lines(contours_lines) + + slope_lines_org_hor=slope_lines_org[slope_lines==0] + args=np.array( range(len(slope_lines) )) + len_x=seperators_closeup.shape[1]/4.0 + + args_hor=args[slope_lines==0] + dist_x_hor=dist_x[slope_lines==0] + x_min_main_hor=x_min_main[slope_lines==0] + x_max_main_hor=x_max_main[slope_lines==0] + cy_main_hor=cy_main[slope_lines==0] + + args_hor=args_hor[dist_x_hor>=len_x/2.0] + x_max_main_hor=x_max_main_hor[dist_x_hor>=len_x/2.0] + x_min_main_hor=x_min_main_hor[dist_x_hor>=len_x/2.0] + cy_main_hor=cy_main_hor[dist_x_hor>=len_x/2.0] + slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x/2.0] + + + slope_lines_org_hor=slope_lines_org_hor[np.abs(slope_lines_org_hor)<1.2] + slope_mean_hor=np.mean(slope_lines_org_hor) + + + + args_ver=args[slope_lines==1] + y_min_main_ver=y_min_main[slope_lines==1] + y_max_main_ver=y_max_main[slope_lines==1] + x_min_main_ver=x_min_main[slope_lines==1] + x_max_main_ver=x_max_main[slope_lines==1] + cx_main_ver=cx_main[slope_lines==1] + dist_y_ver=y_max_main_ver-y_min_main_ver + len_y=seperators_closeup.shape[0]/3.0 + + + + print(matrix_of_lines_ch[:,8][matrix_of_lines_ch[:,9]==0],'khatlarrrr') + args_main_spliters=matrix_of_lines_ch[:,0][ (matrix_of_lines_ch[:,9]==0) & ((matrix_of_lines_ch[:,8]<=290)) & ((matrix_of_lines_ch[:,2]<=.16*region_pre_p.shape[1])) & ((matrix_of_lines_ch[:,3]>=.84*region_pre_p.shape[1]))] + + cy_main_spliters=matrix_of_lines_ch[:,5][ (matrix_of_lines_ch[:,9]==0) & ((matrix_of_lines_ch[:,8]<=290)) & ((matrix_of_lines_ch[:,2]<=.16*region_pre_p.shape[1])) & ((matrix_of_lines_ch[:,3]>=.84*region_pre_p.shape[1]))] + """ + + cy_main_spliters = cy_main_hor[(x_min_main_hor <= 0.16 * region_pre_p.shape[1]) & (x_max_main_hor >= 0.84 * region_pre_p.shape[1])] + + cy_main_spliters = np.array(list(cy_main_spliters) + list(special_seperators)) + + if contours_h is not None: + try: + cy_main_spliters_head = cy_main_head[(x_min_main_head <= 0.16 * region_pre_p.shape[1]) & (x_max_main_head >= 0.84 * region_pre_p.shape[1])] + cy_main_spliters = np.array(list(cy_main_spliters) + list(cy_main_spliters_head)) + except: + pass + args_cy_spliter = np.argsort(cy_main_spliters) + + cy_main_spliters_sort = cy_main_spliters[args_cy_spliter] + + spliter_y_new = [] + spliter_y_new.append(0) + for i in range(len(cy_main_spliters_sort)): + spliter_y_new.append(cy_main_spliters_sort[i]) + + spliter_y_new.append(region_pre_p.shape[0]) + + spliter_y_new_diff = np.diff(spliter_y_new) / float(region_pre_p.shape[0]) * 100 + + args_big_parts = np.array(range(len(spliter_y_new_diff)))[spliter_y_new_diff > 22] + + regions_without_seperators = return_regions_without_seperators(region_pre_p) + + ##print(args_big_parts,'args_big_parts') + # image_page_otsu=otsu_copy(image_page_deskewd) + # print(np.unique(image_page_otsu[:,:,0])) + # image_page_background_zero=self.image_change_background_pixels_to_zero(image_page_otsu) + + length_y_threshold = regions_without_seperators.shape[0] / 4.0 + + num_col_fin = 0 + peaks_neg_fin_fin = [] + + for iteils in args_big_parts: + + regions_without_seperators_teil = regions_without_seperators[int(spliter_y_new[iteils]) : int(spliter_y_new[iteils + 1]), :, 0] + # image_page_background_zero_teil=image_page_background_zero[int(spliter_y_new[iteils]):int(spliter_y_new[iteils+1]),:] + + # print(regions_without_seperators_teil.shape) + ##plt.imshow(regions_without_seperators_teil) + ##plt.show() + + # num_col, peaks_neg_fin=find_num_col(regions_without_seperators_teil,multiplier=6.0) + + # regions_without_seperators_teil=cv2.erode(regions_without_seperators_teil,kernel,iterations = 3) + # + num_col, peaks_neg_fin = find_num_col(regions_without_seperators_teil, multiplier=7.0) + + if num_col > num_col_fin: + num_col_fin = num_col + peaks_neg_fin_fin = peaks_neg_fin + """ + #print(length_y_vertical_lines,length_y_threshold,'x_center_of_ver_linesx_center_of_ver_linesx_center_of_ver_lines') + if len(cx_main_ver)>0 and len( dist_y_ver[dist_y_ver>=length_y_threshold] ) >=1: + num_col, peaks_neg_fin=find_num_col(regions_without_seperators_teil,multiplier=6.0) + else: + #plt.imshow(image_page_background_zero_teil) + #plt.show() + #num_col, peaks_neg_fin=find_num_col_only_image(image_page_background_zero,multiplier=2.4)#2.3) + num_col, peaks_neg_fin=find_num_col_only_image(image_page_background_zero_teil,multiplier=3.4)#2.3) + + print(num_col,'birda') + if num_col>0: + pass + elif num_col==0: + print(num_col,'birda2222') + num_col_regions, peaks_neg_fin_regions=find_num_col(regions_without_seperators_teil,multiplier=10.0) + if num_col_regions==0: + pass + else: + + num_col=num_col_regions + peaks_neg_fin=peaks_neg_fin_regions[:] + """ + + # print(num_col+1,'num colmsssssssss') + + if len(args_big_parts) == 1 and (len(peaks_neg_fin_fin) + 1) < num_col_classifier: + peaks_neg_fin = find_num_col_by_vertical_lines(vertical) + peaks_neg_fin = peaks_neg_fin[peaks_neg_fin >= 500] + peaks_neg_fin = peaks_neg_fin[peaks_neg_fin <= (vertical.shape[1] - 500)] + peaks_neg_fin_fin = peaks_neg_fin[:] + + # print(peaks_neg_fin_fin,'peaks_neg_fin_fintaza') + + return num_col_fin, peaks_neg_fin_fin, matrix_of_lines_ch, spliter_y_new, seperators_closeup_n + +def return_boxes_of_images_by_order_of_reading_new(spliter_y_new, regions_without_seperators, matrix_of_lines_ch): + boxes = [] + + # here I go through main spliters and i do check whether a vertical seperator there is. If so i am searching for \ + # holes in the text and also finding spliter which covers more than one columns. + for i in range(len(spliter_y_new) - 1): + # print(spliter_y_new[i],spliter_y_new[i+1]) + matrix_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 6] > spliter_y_new[i]) & (matrix_of_lines_ch[:, 7] < spliter_y_new[i + 1])] + # print(len( matrix_new[:,9][matrix_new[:,9]==1] )) + + # print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') + + # check to see is there any vertical seperator to find holes. + if 1 > 0: # len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(spliter_y_new[i+1]-spliter_y_new[i] )): + + # org_img_dichte=-gaussian_filter1d(( image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,0]/255.).sum(axis=0) ,30) + # org_img_dichte=org_img_dichte-np.min(org_img_dichte) + ##plt.figure(figsize=(20,20)) + ##plt.plot(org_img_dichte) + ##plt.show() + ###find_num_col_both_layout_and_org(regions_without_seperators,image_page[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:,:],7.) + + # print(int(spliter_y_new[i]),int(spliter_y_new[i+1]),'firssst') + + # plt.imshow(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:]) + # plt.show() + try: + num_col, peaks_neg_fin = find_num_col(regions_without_seperators[int(spliter_y_new[i]) : int(spliter_y_new[i + 1]), :], multiplier=7.0) + except: + peaks_neg_fin = [] + + # print(peaks_neg_fin,'peaks_neg_fin') + # num_col, peaks_neg_fin=find_num_col(regions_without_seperators[int(spliter_y_new[i]):int(spliter_y_new[i+1]),:],multiplier=7.0) + x_min_hor_some = matrix_new[:, 2][(matrix_new[:, 9] == 0)] + x_max_hor_some = matrix_new[:, 3][(matrix_new[:, 9] == 0)] + cy_hor_some = matrix_new[:, 5][(matrix_new[:, 9] == 0)] + arg_org_hor_some = matrix_new[:, 0][(matrix_new[:, 9] == 0)] + + peaks_neg_tot = return_points_with_boundies(peaks_neg_fin, 0, regions_without_seperators[:, :].shape[1]) + + start_index_of_hor, newest_peaks, arg_min_hor_sort, lines_length_dels, lines_indexes_deleted = return_hor_spliter_by_index_for_without_verticals(peaks_neg_tot, x_min_hor_some, x_max_hor_some) + + arg_org_hor_some_sort = arg_org_hor_some[arg_min_hor_sort] + + start_index_of_hor_with_subset = [start_index_of_hor[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] # start_index_of_hor[lines_length_dels>0] + arg_min_hor_sort_with_subset = [arg_min_hor_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + lines_indexes_deleted_with_subset = [lines_indexes_deleted[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + lines_length_dels_with_subset = [lines_length_dels[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + + arg_org_hor_some_sort_subset = [arg_org_hor_some_sort[vij] for vij in range(len(start_index_of_hor)) if lines_length_dels[vij] > 0] + + # arg_min_hor_sort_with_subset=arg_min_hor_sort[lines_length_dels>0] + # lines_indexes_deleted_with_subset=lines_indexes_deleted[lines_length_dels>0] + # lines_length_dels_with_subset=lines_length_dels[lines_length_dels>0] + + vahid_subset = np.zeros((len(start_index_of_hor_with_subset), len(start_index_of_hor_with_subset))) - 1 + for kkk1 in range(len(start_index_of_hor_with_subset)): + + index_del_sub = np.unique(lines_indexes_deleted_with_subset[kkk1]) + + for kkk2 in range(len(start_index_of_hor_with_subset)): + + if set(lines_indexes_deleted_with_subset[kkk2][0]) < set(lines_indexes_deleted_with_subset[kkk1][0]): + vahid_subset[kkk1, kkk2] = kkk1 + else: + pass + # print(set(lines_indexes_deleted[kkk2][0]), set(lines_indexes_deleted[kkk1][0])) + + # check the len of matrix if it has no length means that there is no spliter at all + + if len(vahid_subset > 0): + # print('hihoo') + + # find parenets args + line_int = np.zeros(vahid_subset.shape[0]) + + childs_id = [] + arg_child = [] + for li in range(vahid_subset.shape[0]): + # print(vahid_subset[:,li]) + if np.all(vahid_subset[:, li] == -1): + line_int[li] = -1 + else: + line_int[li] = 1 + + # childs_args_in=[ idd for idd in range(vahid_subset.shape[0]) if vahid_subset[idd,li]!=-1] + # helpi=[] + # for nad in range(len(childs_args_in)): + # helpi.append(arg_min_hor_sort_with_subset[childs_args_in[nad]]) + + arg_child.append(arg_min_hor_sort_with_subset[li]) + + # line_int=vahid_subset[0,:] + + arg_parent = [arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] == -1] + start_index_of_hor_parent = [start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] == -1] + # arg_parent=[lines_indexes_deleted_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] + # arg_parent=[lines_length_dels_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]==-1] + + # arg_child=[arg_min_hor_sort_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij]!=-1] + start_index_of_hor_child = [start_index_of_hor_with_subset[vij] for vij in range(len(arg_min_hor_sort_with_subset)) if line_int[vij] != -1] + + cy_hor_some_sort = cy_hor_some[arg_parent] + + # print(start_index_of_hor, lines_length_dels ,lines_indexes_deleted,'zartt') + + # args_indexes=np.array(range(len(start_index_of_hor) )) + + newest_y_spliter_tot = [] + + for tj in range(len(newest_peaks) - 1): + newest_y_spliter = [] + newest_y_spliter.append(spliter_y_new[i]) + if tj in np.unique(start_index_of_hor_parent): + # print(cy_hor_some_sort) + cy_help = np.array(cy_hor_some_sort)[np.array(start_index_of_hor_parent) == tj] + cy_help_sort = np.sort(cy_help) + + # print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') + for mj in range(len(cy_help_sort)): + newest_y_spliter.append(cy_help_sort[mj]) + newest_y_spliter.append(spliter_y_new[i + 1]) + + newest_y_spliter_tot.append(newest_y_spliter) + + else: + line_int = [] + newest_y_spliter_tot = [] + + for tj in range(len(newest_peaks) - 1): + newest_y_spliter = [] + newest_y_spliter.append(spliter_y_new[i]) + + newest_y_spliter.append(spliter_y_new[i + 1]) + + newest_y_spliter_tot.append(newest_y_spliter) + + # if line_int is all -1 means that big spliters have no child and we can easily go through + if np.all(np.array(line_int) == -1): + for j in range(len(newest_peaks) - 1): + newest_y_spliter = newest_y_spliter_tot[j] + + for n in range(len(newest_y_spliter) - 1): + # print(j,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'maaaa') + ##plt.imshow(regions_without_seperators[int(newest_y_spliter[n]):int(newest_y_spliter[n+1]),newest_peaks[j]:newest_peaks[j+1]]) + ##plt.show() + + # print(matrix_new[:,0][ (matrix_new[:,9]==1 )]) + for jvt in matrix_new[:, 0][(matrix_new[:, 9] == 1) & (matrix_new[:, 6] > newest_y_spliter[n]) & (matrix_new[:, 7] < newest_y_spliter[n + 1]) & ((matrix_new[:, 1]) < newest_peaks[j + 1]) & ((matrix_new[:, 1]) > newest_peaks[j])]: + pass + + ###plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) + # print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): + # print( int(newest_y_spliter[n]),int(newest_y_spliter[n+1]),newest_peaks[j],newest_peaks[j+1] ) + try: + num_col_sub, peaks_neg_fin_sub = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=7.0) + except: + peaks_neg_fin_sub = [] + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + for j in range(len(newest_peaks) - 1): + + newest_y_spliter = newest_y_spliter_tot[j] + + if j in start_index_of_hor_parent: + + x_min_ch = x_min_hor_some[arg_child] + x_max_ch = x_max_hor_some[arg_child] + cy_hor_some_sort_child = cy_hor_some[arg_child] + cy_hor_some_sort_child = np.sort(cy_hor_some_sort_child) + + for n in range(len(newest_y_spliter) - 1): + + cy_child_in = cy_hor_some_sort_child[(cy_hor_some_sort_child > newest_y_spliter[n]) & (cy_hor_some_sort_child < newest_y_spliter[n + 1])] + + if len(cy_child_in) > 0: + try: + num_col_ch, peaks_neg_ch = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=7.0) + except: + peaks_neg_ch = [] + # print(peaks_neg_ch,'mizzzz') + # peaks_neg_ch=[] + # for djh in range(len(peaks_neg_ch)): + # peaks_neg_ch.append( peaks_neg_ch[djh]+newest_peaks[j] ) + + peaks_neg_ch_tot = return_points_with_boundies(peaks_neg_ch, newest_peaks[j], newest_peaks[j + 1]) + + ss_in_ch, nst_p_ch, arg_n_ch, lines_l_del_ch, lines_in_del_ch = return_hor_spliter_by_index_for_without_verticals(peaks_neg_ch_tot, x_min_ch, x_max_ch) + + newest_y_spliter_ch_tot = [] + + for tjj in range(len(nst_p_ch) - 1): + newest_y_spliter_new = [] + newest_y_spliter_new.append(newest_y_spliter[n]) + if tjj in np.unique(ss_in_ch): + + # print(tj,cy_hor_some_sort,start_index_of_hor,cy_help,'maashhaha') + for mjj in range(len(cy_child_in)): + newest_y_spliter_new.append(cy_child_in[mjj]) + newest_y_spliter_new.append(newest_y_spliter[n + 1]) + + newest_y_spliter_ch_tot.append(newest_y_spliter_new) + + for jn in range(len(nst_p_ch) - 1): + newest_y_spliter_h = newest_y_spliter_ch_tot[jn] + + for nd in range(len(newest_y_spliter_h) - 1): + + matrix_new_new2 = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter_h[nd]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter_h[nd + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < nst_p_ch[jn + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > nst_p_ch[jn])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new2[:,9][matrix_new_new2[:,9]==1] )>0 and np.max(matrix_new_new2[:,8][matrix_new_new2[:,9]==1])>=0.2*(np.abs(newest_y_spliter_h[nd+1]-newest_y_spliter_h[nd] )): + try: + num_col_sub_ch, peaks_neg_fin_sub_ch = find_num_col(regions_without_seperators[int(newest_y_spliter_h[nd]) : int(newest_y_spliter_h[nd + 1]), nst_p_ch[jn] : nst_p_ch[jn + 1]], multiplier=7.0) + except: + peaks_neg_fin_sub_ch = [] + + else: + peaks_neg_fin_sub_ch = [] + + peaks_sub_ch = [] + peaks_sub_ch.append(nst_p_ch[jn]) + + for kjj in range(len(peaks_neg_fin_sub_ch)): + peaks_sub_ch.append(peaks_neg_fin_sub_ch[kjj] + nst_p_ch[jn]) + + peaks_sub_ch.append(nst_p_ch[jn + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for khh in range(len(peaks_sub_ch) - 1): + boxes.append([peaks_sub_ch[khh], peaks_sub_ch[khh + 1], newest_y_spliter_h[nd], newest_y_spliter_h[nd + 1]]) + + else: + + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): + try: + num_col_sub, peaks_neg_fin_sub = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=7.0) + except: + peaks_neg_fin_sub = [] + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + for n in range(len(newest_y_spliter) - 1): + + # plot_contour(regions_without_seperators.shape[0],regions_without_seperators.shape[1], contours_lines[int(jvt)]) + # print(matrix_of_lines_ch[matrix_of_lines_ch[:,9]==1]) + matrix_new_new = matrix_of_lines_ch[:, :][(matrix_of_lines_ch[:, 9] == 1) & (matrix_of_lines_ch[:, 6] > newest_y_spliter[n]) & (matrix_of_lines_ch[:, 7] < newest_y_spliter[n + 1]) & ((matrix_of_lines_ch[:, 1] + 500) < newest_peaks[j + 1]) & ((matrix_of_lines_ch[:, 1] - 500) > newest_peaks[j])] + # print(matrix_new_new,newest_y_spliter[n],newest_y_spliter[n+1],newest_peaks[j],newest_peaks[j+1],'gada') + if 1 > 0: # len( matrix_new_new[:,9][matrix_new_new[:,9]==1] )>0 and np.max(matrix_new_new[:,8][matrix_new_new[:,9]==1])>=0.2*(np.abs(newest_y_spliter[n+1]-newest_y_spliter[n] )): + try: + num_col_sub, peaks_neg_fin_sub = find_num_col(regions_without_seperators[int(newest_y_spliter[n]) : int(newest_y_spliter[n + 1]), newest_peaks[j] : newest_peaks[j + 1]], multiplier=5.0) + except: + peaks_neg_fin_sub = [] + else: + peaks_neg_fin_sub = [] + + peaks_sub = [] + peaks_sub.append(newest_peaks[j]) + + for kj in range(len(peaks_neg_fin_sub)): + peaks_sub.append(peaks_neg_fin_sub[kj] + newest_peaks[j]) + + peaks_sub.append(newest_peaks[j + 1]) + + # peaks_sub=return_points_with_boundies(peaks_neg_fin_sub+newest_peaks[j],newest_peaks[j], newest_peaks[j+1]) + + for kh in range(len(peaks_sub) - 1): + boxes.append([peaks_sub[kh], peaks_sub[kh + 1], newest_y_spliter[n], newest_y_spliter[n + 1]]) + + else: + boxes.append([0, regions_without_seperators[:, :].shape[1], spliter_y_new[i], spliter_y_new[i + 1]]) + + return boxes + diff --git a/sbb_newspapers_org_image/utils/contour.py b/sbb_newspapers_org_image/utils/contour.py new file mode 100644 index 0000000..7c8a283 --- /dev/null +++ b/sbb_newspapers_org_image/utils/contour.py @@ -0,0 +1,298 @@ +import cv2 +import numpy as np +from shapely import geometry + +from .rotate import rotate_image, rotation_image_new + +def contours_in_same_horizon(cy_main_hor): + X1 = np.zeros((len(cy_main_hor), len(cy_main_hor))) + X2 = np.zeros((len(cy_main_hor), len(cy_main_hor))) + + X1[0::1, :] = cy_main_hor[:] + X2 = X1.T + + X_dif = np.abs(X2 - X1) + args_help = np.array(range(len(cy_main_hor))) + all_args = [] + for i in range(len(cy_main_hor)): + list_h = list(args_help[X_dif[i, :] <= 20]) + list_h.append(i) + if len(list_h) > 1: + all_args.append(list(set(list_h))) + return np.unique(all_args) + +def find_contours_mean_y_diff(contours_main): + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + return np.mean(np.diff(np.sort(np.array(cy_main)))) + +def find_features_of_contours(contours_main): + + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + + return y_min_main, y_max_main, areas_main + +def return_contours_of_interested_region_and_bounding_box(region_pre_p, pixel): + + # pixels of images are identified by 5 + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_imgs, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hiearchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hiearchy, max_area=1, min_area=0.0003) + + boxes = [] + + for jj in range(len(contours_imgs)): + x, y, w, h = cv2.boundingRect(contours_imgs[jj]) + boxes.append([int(x), int(y), int(w), int(h)]) + return contours_imgs, boxes + +def get_text_region_boxes_by_given_contours(contours): + + kernel = np.ones((5, 5), np.uint8) + boxes = [] + contours_new = [] + for jj in range(len(contours)): + x, y, w, h = cv2.boundingRect(contours[jj]) + + boxes.append([x, y, w, h]) + contours_new.append(contours[jj]) + + del contours + return boxes, contours_new + +def filter_contours_area_of_image(image, contours, hirarchy, max_area, min_area): + found_polygons_early = list() + + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + area = polygon.area + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and hirarchy[0][jv][3] == -1: # and hirarchy[0][jv][3]==-1 : + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.uint)) + jv += 1 + return found_polygons_early + +def filter_contours_area_of_image_interiors(image, contours, hirarchy, max_area, min_area): + found_polygons_early = list() + + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + area = polygon.area + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and hirarchy[0][jv][3] != -1: + # print(c[0][0][1]) + found_polygons_early.append(np.array([point for point in polygon.exterior.coords], dtype=np.uint)) + jv += 1 + return found_polygons_early + + +def filter_contours_area_of_image_tables(image, contours, hirarchy, max_area, min_area): + found_polygons_early = list() + + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + # area = cv2.contourArea(c) + area = polygon.area + ##print(np.prod(thresh.shape[:2])) + # Check that polygon has area greater than minimal area + # print(hirarchy[0][jv][3],hirarchy ) + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hirarchy[0][jv][3]==-1 : + # print(c[0][0][1]) + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.int32)) + jv += 1 + return found_polygons_early + +def find_new_features_of_contoures(contours_main): + + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + try: + x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + argmin_x_main = np.array([np.argmin(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 0] for j in range(len(contours_main))]) + y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 1] for j in range(len(contours_main))]) + + x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + except: + x_min_main = np.array([np.min(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + argmin_x_main = np.array([np.argmin(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0] for j in range(len(contours_main))]) + y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 1] for j in range(len(contours_main))]) + + x_max_main = np.array([np.max(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 1]) for j in range(len(contours_main))]) + + # dis_x=np.abs(x_max_main-x_min_main) + + return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin + +def return_parent_contours(contours, hierarchy): + contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1] + return contours_parent + +def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_imgs, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hiearchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hiearchy, max_area=1, min_area=min_area) + + return contours_imgs + +def get_textregion_contours_in_org_image(cnts, img, slope_first): + + cnts_org = [] + # print(cnts,'cnts') + for i in range(len(cnts)): + img_copy = np.zeros(img.shape) + img_copy = cv2.fillPoly(img_copy, pts=[cnts[i]], color=(1, 1, 1)) + + # plt.imshow(img_copy) + # plt.show() + + # print(img.shape,'img') + img_copy = rotation_image_new(img_copy, -slope_first) + ##print(img_copy.shape,'img_copy') + # plt.imshow(img_copy) + # plt.show() + + img_copy = img_copy.astype(np.uint8) + imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) + cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) + # print(np.shape(cont_int[0])) + cnts_org.append(cont_int[0]) + + # print(cnts_org,'cnts_org') + + # sys.exit() + # self.y_shift = np.abs(img_copy.shape[0] - img.shape[0]) + # self.x_shift = np.abs(img_copy.shape[1] - img.shape[1]) + return cnts_org + +def return_contours_of_interested_textline(region_pre_p, pixel): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_imgs, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hiearchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hiearchy, max_area=1, min_area=0.000000003) + return contours_imgs + +def return_bonding_box_of_contours(cnts): + boxes_tot = [] + for i in range(len(cnts)): + x, y, w, h = cv2.boundingRect(cnts[i]) + + box = [x, y, w, h] + boxes_tot.append(box) + return boxes_tot + +def return_contours_of_image(image): + + if len(image.shape) == 2: + image = np.repeat(image[:, :, np.newaxis], 3, axis=2) + image = image.astype(np.uint8) + else: + image = image.astype(np.uint8) + imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + return contours, hierachy + +def return_contours_of_interested_region_by_min_size(region_pre_p, pixel, min_size=0.00003): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_imgs, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hiearchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hiearchy, max_area=1, min_area=min_size) + + return contours_imgs + +def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, max_area): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_imgs, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hiearchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hiearchy, max_area=max_area, min_area=min_area) + + img_ret = np.zeros((region_pre_p.shape[0], region_pre_p.shape[1], 3)) + img_ret = cv2.fillPoly(img_ret, pts=contours_imgs, color=(1, 1, 1)) + return img_ret[:, :, 0] + diff --git a/sbb_newspapers_org_image/utils/drop_capitals.py b/sbb_newspapers_org_image/utils/drop_capitals.py new file mode 100644 index 0000000..5479457 --- /dev/null +++ b/sbb_newspapers_org_image/utils/drop_capitals.py @@ -0,0 +1,501 @@ +import numpy as np +import cv2 +from .contour import ( + find_new_features_of_contoures, + return_contours_of_image, + return_parent_contours, +) + +def adhere_drop_capital_region_into_cprresponding_textline( + text_regions_p, + polygons_of_drop_capitals, + contours_only_text_parent, + contours_only_text_parent_h, + all_box_coord, + all_box_coord_h, + all_found_texline_polygons, + all_found_texline_polygons_h, + kernel=None, + curved_line=False, +): + # print(np.shape(all_found_texline_polygons),np.shape(all_found_texline_polygons[3]),'all_found_texline_polygonsshape') + # print(all_found_texline_polygons[3]) + cx_m, cy_m, _, _, _, _, _ = find_new_features_of_contoures(contours_only_text_parent) + cx_h, cy_h, _, _, _, _, _ = find_new_features_of_contoures(contours_only_text_parent_h) + cx_d, cy_d, _, _, y_min_d, y_max_d, _ = find_new_features_of_contoures(polygons_of_drop_capitals) + + img_con_all = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + for j_cont in range(len(contours_only_text_parent)): + img_con_all[all_box_coord[j_cont][0] : all_box_coord[j_cont][1], all_box_coord[j_cont][2] : all_box_coord[j_cont][3], 0] = (j_cont + 1) * 3 + # img_con_all=cv2.fillPoly(img_con_all,pts=[contours_only_text_parent[j_cont]],color=((j_cont+1)*3,(j_cont+1)*3,(j_cont+1)*3)) + + # plt.imshow(img_con_all[:,:,0]) + # plt.show() + # img_con_all=cv2.dilate(img_con_all, kernel, iterations=3) + + # plt.imshow(img_con_all[:,:,0]) + # plt.show() + # print(np.unique(img_con_all[:,:,0])) + for i_drop in range(len(polygons_of_drop_capitals)): + # print(i_drop,'i_drop') + img_con_all_copy = np.copy(img_con_all) + img_con = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_con = cv2.fillPoly(img_con, pts=[polygons_of_drop_capitals[i_drop]], color=(1, 1, 1)) + + # plt.imshow(img_con[:,:,0]) + # plt.show() + ##img_con=cv2.dilate(img_con, kernel, iterations=30) + + # plt.imshow(img_con[:,:,0]) + # plt.show() + + # print(np.unique(img_con[:,:,0])) + + img_con_all_copy[:, :, 0] = img_con_all_copy[:, :, 0] + img_con[:, :, 0] + + img_con_all_copy[:, :, 0][img_con_all_copy[:, :, 0] == 1] = 0 + + kherej_ghesmat = np.unique(img_con_all_copy[:, :, 0]) / 3 + res_summed_pixels = np.unique(img_con_all_copy[:, :, 0]) % 3 + region_with_intersected_drop = kherej_ghesmat[res_summed_pixels == 1] + # region_with_intersected_drop=region_with_intersected_drop/3 + region_with_intersected_drop = region_with_intersected_drop.astype(np.uint8) + + # print(len(region_with_intersected_drop),'region_with_intersected_drop1') + if len(region_with_intersected_drop) == 0: + img_con_all_copy = np.copy(img_con_all) + img_con = cv2.dilate(img_con, kernel, iterations=4) + + img_con_all_copy[:, :, 0] = img_con_all_copy[:, :, 0] + img_con[:, :, 0] + + img_con_all_copy[:, :, 0][img_con_all_copy[:, :, 0] == 1] = 0 + + kherej_ghesmat = np.unique(img_con_all_copy[:, :, 0]) / 3 + res_summed_pixels = np.unique(img_con_all_copy[:, :, 0]) % 3 + region_with_intersected_drop = kherej_ghesmat[res_summed_pixels == 1] + # region_with_intersected_drop=region_with_intersected_drop/3 + region_with_intersected_drop = region_with_intersected_drop.astype(np.uint8) + # print(np.unique(img_con_all_copy[:,:,0])) + if curved_line: + + if len(region_with_intersected_drop) > 1: + sum_pixels_of_intersection = [] + for i in range(len(region_with_intersected_drop)): + # print((region_with_intersected_drop[i]*3+1)) + sum_pixels_of_intersection.append(((img_con_all_copy[:, :, 0] == (region_with_intersected_drop[i] * 3 + 1)) * 1).sum()) + # print(sum_pixels_of_intersection) + region_final = region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)] - 1 + + # print(region_final,'region_final') + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + try: + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + # print(all_box_coord[j_cont]) + # print(cx_t) + # print(cy_t) + # print(cx_d[i_drop]) + # print(cy_d[i_drop]) + y_lines = np.array(cy_t) # all_box_coord[int(region_final)][0]+np.array(cy_t) + + # print(y_lines) + + y_lines[y_lines < y_min_d[i_drop]] = 0 + # print(y_lines) + + arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) + # print(arg_min) + + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] + cnt_nearest[:, 0, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] + + img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) + img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) + + img_textlines = img_textlines.astype(np.uint8) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_combined, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(len(contours_combined),'len textlines mixed') + areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + contours_biggest = contours_combined[np.argmax(areas_cnt_text)] + + # print(np.shape(contours_biggest)) + # print(contours_biggest[:]) + # contours_biggest[:,0,0]=contours_biggest[:,0,0]#-all_box_coord[int(region_final)][2] + # contours_biggest[:,0,1]=contours_biggest[:,0,1]#-all_box_coord[int(region_final)][0] + + # contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) + + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + + except: + # print('gordun1') + pass + elif len(region_with_intersected_drop) == 1: + region_final = region_with_intersected_drop[0] - 1 + + # areas_main=np.array([cv2.contourArea(all_found_texline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_texline_polygons[int(region_final)]))]) + + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + # print(all_box_coord[j_cont]) + # print(cx_t) + # print(cy_t) + # print(cx_d[i_drop]) + # print(cy_d[i_drop]) + y_lines = np.array(cy_t) # all_box_coord[int(region_final)][0]+np.array(cy_t) + + y_lines[y_lines < y_min_d[i_drop]] = 0 + # print(y_lines) + + arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) + # print(arg_min) + + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] + cnt_nearest[:, 0, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] + + img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) + img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) + + img_textlines = img_textlines.astype(np.uint8) + + # plt.imshow(img_textlines) + # plt.show() + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_combined, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(len(contours_combined),'len textlines mixed') + areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + contours_biggest = contours_combined[np.argmax(areas_cnt_text)] + + # print(np.shape(contours_biggest)) + # print(contours_biggest[:]) + # contours_biggest[:,0,0]=contours_biggest[:,0,0]#-all_box_coord[int(region_final)][2] + # contours_biggest[:,0,1]=contours_biggest[:,0,1]#-all_box_coord[int(region_final)][0] + # print(np.shape(contours_biggest),'contours_biggest') + # print(np.shape(all_found_texline_polygons[int(region_final)][arg_min])) + ##contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + + # print(cx_t,'print') + try: + # print(all_found_texline_polygons[j_cont][0]) + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + # print(all_box_coord[j_cont]) + # print(cx_t) + # print(cy_t) + # print(cx_d[i_drop]) + # print(cy_d[i_drop]) + y_lines = all_box_coord[int(region_final)][0] + np.array(cy_t) + + y_lines[y_lines < y_min_d[i_drop]] = 0 + # print(y_lines) + + arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) + # print(arg_min) + + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] + cnt_nearest[:, 0, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] + + img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) + img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) + + img_textlines = img_textlines.astype(np.uint8) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_combined, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(len(contours_combined),'len textlines mixed') + areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + contours_biggest = contours_combined[np.argmax(areas_cnt_text)] + + # print(np.shape(contours_biggest)) + # print(contours_biggest[:]) + contours_biggest[:, 0, 0] = contours_biggest[:, 0, 0] # -all_box_coord[int(region_final)][2] + contours_biggest[:, 0, 1] = contours_biggest[:, 0, 1] # -all_box_coord[int(region_final)][0] + + ##contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + # all_found_texline_polygons[int(region_final)][arg_min]=contours_biggest + + except: + pass + else: + pass + + ##cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + ###print(all_box_coord[j_cont]) + ###print(cx_t) + ###print(cy_t) + ###print(cx_d[i_drop]) + ###print(cy_d[i_drop]) + ##y_lines=all_box_coord[int(region_final)][0]+np.array(cy_t) + + ##y_lines[y_lines 1: + sum_pixels_of_intersection = [] + for i in range(len(region_with_intersected_drop)): + # print((region_with_intersected_drop[i]*3+1)) + sum_pixels_of_intersection.append(((img_con_all_copy[:, :, 0] == (region_with_intersected_drop[i] * 3 + 1)) * 1).sum()) + # print(sum_pixels_of_intersection) + region_final = region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)] - 1 + + # print(region_final,'region_final') + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + try: + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + # print(all_box_coord[j_cont]) + # print(cx_t) + # print(cy_t) + # print(cx_d[i_drop]) + # print(cy_d[i_drop]) + y_lines = all_box_coord[int(region_final)][0] + np.array(cy_t) + + # print(y_lines) + + y_lines[y_lines < y_min_d[i_drop]] = 0 + # print(y_lines) + + arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) + # print(arg_min) + + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0] + all_box_coord[int(region_final)][2] + cnt_nearest[:, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 1] + all_box_coord[int(region_final)][0] + + img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) + img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) + + img_textlines = img_textlines.astype(np.uint8) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_combined, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(len(contours_combined),'len textlines mixed') + areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + contours_biggest = contours_combined[np.argmax(areas_cnt_text)] + + # print(np.shape(contours_biggest)) + # print(contours_biggest[:]) + contours_biggest[:, 0, 0] = contours_biggest[:, 0, 0] - all_box_coord[int(region_final)][2] + contours_biggest[:, 0, 1] = contours_biggest[:, 0, 1] - all_box_coord[int(region_final)][0] + + contours_biggest = contours_biggest.reshape(np.shape(contours_biggest)[0], np.shape(contours_biggest)[2]) + + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + + except: + # print('gordun1') + pass + elif len(region_with_intersected_drop) == 1: + region_final = region_with_intersected_drop[0] - 1 + + # areas_main=np.array([cv2.contourArea(all_found_texline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_texline_polygons[int(region_final)]))]) + + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + + # print(cx_t,'print') + try: + # print(all_found_texline_polygons[j_cont][0]) + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contoures(all_found_texline_polygons[int(region_final)]) + # print(all_box_coord[j_cont]) + # print(cx_t) + # print(cy_t) + # print(cx_d[i_drop]) + # print(cy_d[i_drop]) + y_lines = all_box_coord[int(region_final)][0] + np.array(cy_t) + + y_lines[y_lines < y_min_d[i_drop]] = 0 + # print(y_lines) + + arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) + # print(arg_min) + + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0] + all_box_coord[int(region_final)][2] + cnt_nearest[:, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 1] + all_box_coord[int(region_final)][0] + + img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) + img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) + + img_textlines = img_textlines.astype(np.uint8) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_combined, hierachy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(len(contours_combined),'len textlines mixed') + areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + contours_biggest = contours_combined[np.argmax(areas_cnt_text)] + + # print(np.shape(contours_biggest)) + # print(contours_biggest[:]) + contours_biggest[:, 0, 0] = contours_biggest[:, 0, 0] - all_box_coord[int(region_final)][2] + contours_biggest[:, 0, 1] = contours_biggest[:, 0, 1] - all_box_coord[int(region_final)][0] + + contours_biggest = contours_biggest.reshape(np.shape(contours_biggest)[0], np.shape(contours_biggest)[2]) + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + # all_found_texline_polygons[int(region_final)][arg_min]=contours_biggest + + except: + pass + else: + pass + + #####for i_drop in range(len(polygons_of_drop_capitals)): + #####for j_cont in range(len(contours_only_text_parent)): + #####img_con=np.zeros((text_regions_p.shape[0],text_regions_p.shape[1],3)) + #####img_con=cv2.fillPoly(img_con,pts=[polygons_of_drop_capitals[i_drop] ],color=(255,255,255)) + #####img_con=cv2.fillPoly(img_con,pts=[contours_only_text_parent[j_cont]],color=(255,255,255)) + + #####img_con=img_con.astype(np.uint8) + ######imgray = cv2.cvtColor(img_con, cv2.COLOR_BGR2GRAY) + ######ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + ######contours_new,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + + #####contours_new,hir_new=return_contours_of_image(img_con) + #####contours_new_parent=return_parent_contours( contours_new,hir_new) + ######plt.imshow(img_con) + ######plt.show() + #####try: + #####if len(contours_new_parent)==1: + ######print(all_found_texline_polygons[j_cont][0]) + #####cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contoures(all_found_texline_polygons[j_cont]) + ######print(all_box_coord[j_cont]) + ######print(cx_t) + ######print(cy_t) + ######print(cx_d[i_drop]) + ######print(cy_d[i_drop]) + #####y_lines=all_box_coord[j_cont][0]+np.array(cy_t) + + ######print(y_lines) + + #####arg_min=np.argmin(np.abs(y_lines-y_min_d[i_drop]) ) + ######print(arg_min) + + #####cnt_nearest=np.copy(all_found_texline_polygons[j_cont][arg_min]) + #####cnt_nearest[:,0]=all_found_texline_polygons[j_cont][arg_min][:,0]+all_box_coord[j_cont][2] + #####cnt_nearest[:,1]=all_found_texline_polygons[j_cont][arg_min][:,1]+all_box_coord[j_cont][0] + + #####img_textlines=np.zeros((text_regions_p.shape[0],text_regions_p.shape[1],3)) + #####img_textlines=cv2.fillPoly(img_textlines,pts=[cnt_nearest],color=(255,255,255)) + #####img_textlines=cv2.fillPoly(img_textlines,pts=[polygons_of_drop_capitals[i_drop] ],color=(255,255,255)) + + #####img_textlines=img_textlines.astype(np.uint8) + #####imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + #####ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + #####contours_combined,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + + #####areas_cnt_text=np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + #####contours_biggest=contours_combined[np.argmax(areas_cnt_text)] + + ######print(np.shape(contours_biggest)) + ######print(contours_biggest[:]) + #####contours_biggest[:,0,0]=contours_biggest[:,0,0]-all_box_coord[j_cont][2] + #####contours_biggest[:,0,1]=contours_biggest[:,0,1]-all_box_coord[j_cont][0] + + #####all_found_texline_polygons[j_cont][arg_min]=contours_biggest + ######print(contours_biggest) + ######plt.imshow(img_textlines[:,:,0]) + ######plt.show() + #####else: + #####pass + #####except: + #####pass + return all_found_texline_polygons + +def filter_small_drop_capitals_from_no_patch_layout(layout_no_patch, layout1): + + drop_only = (layout_no_patch[:, :, 0] == 4) * 1 + contours_drop, hir_on_drop = return_contours_of_image(drop_only) + contours_drop_parent = return_parent_contours(contours_drop, hir_on_drop) + + areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) + areas_cnt_text = areas_cnt_text / float(drop_only.shape[0] * drop_only.shape[1]) + + contours_drop_parent = [contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz] > 0.001] + + areas_cnt_text = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > 0.001] + + contours_drop_parent_final = [] + + for jj in range(len(contours_drop_parent)): + x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) + # boxes.append([int(x), int(y), int(w), int(h)]) + + iou_of_box_and_contoure = float(drop_only.shape[0] * drop_only.shape[1]) * areas_cnt_text[jj] / float(w * h) * 100 + height_to_weight_ratio = h / float(w) + weigh_to_height_ratio = w / float(h) + + if iou_of_box_and_contoure > 60 and weigh_to_height_ratio < 1.2 and height_to_weight_ratio < 2: + map_of_drop_contour_bb = np.zeros((layout1.shape[0], layout1.shape[1])) + map_of_drop_contour_bb[y : y + h, x : x + w] = layout1[y : y + h, x : x + w] + + if (((map_of_drop_contour_bb == 1) * 1).sum() / float(((map_of_drop_contour_bb == 5) * 1).sum()) * 100) >= 15: + contours_drop_parent_final.append(contours_drop_parent[jj]) + + layout_no_patch[:, :, 0][layout_no_patch[:, :, 0] == 4] = 0 + + layout_no_patch = cv2.fillPoly(layout_no_patch, pts=contours_drop_parent_final, color=(4, 4, 4)) + + return layout_no_patch + diff --git a/sbb_newspapers_org_image/utils/is_nan.py b/sbb_newspapers_org_image/utils/is_nan.py new file mode 100644 index 0000000..dcd49ba --- /dev/null +++ b/sbb_newspapers_org_image/utils/is_nan.py @@ -0,0 +1,3 @@ + +def isNaN(num): + return num != num diff --git a/sbb_newspapers_org_image/utils/marginals.py b/sbb_newspapers_org_image/utils/marginals.py new file mode 100644 index 0000000..08c700a --- /dev/null +++ b/sbb_newspapers_org_image/utils/marginals.py @@ -0,0 +1,252 @@ +import numpy as np +import cv2 +from scipy.signal import find_peaks +from scipy.ndimage import gaussian_filter1d + + +from .contour import find_new_features_of_contoures, return_contours_of_interested_region +from .resize import resize_image +from .rotate import rotate_image + +def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, kernel=None): + mask_marginals=np.zeros((text_with_lines.shape[0],text_with_lines.shape[1])) + mask_marginals=mask_marginals.astype(np.uint8) + + + text_with_lines=text_with_lines.astype(np.uint8) + ##text_with_lines=cv2.erode(text_with_lines,self.kernel,iterations=3) + + text_with_lines_eroded=cv2.erode(text_with_lines,kernel,iterations=5) + + if text_with_lines.shape[0]<=1500: + pass + elif text_with_lines.shape[0]>1500 and text_with_lines.shape[0]<=1800: + text_with_lines=resize_image(text_with_lines,int(text_with_lines.shape[0]*1.5),text_with_lines.shape[1]) + text_with_lines=cv2.erode(text_with_lines,kernel,iterations=5) + text_with_lines=resize_image(text_with_lines,text_with_lines_eroded.shape[0],text_with_lines_eroded.shape[1]) + else: + text_with_lines=resize_image(text_with_lines,int(text_with_lines.shape[0]*1.8),text_with_lines.shape[1]) + text_with_lines=cv2.erode(text_with_lines,kernel,iterations=7) + text_with_lines=resize_image(text_with_lines,text_with_lines_eroded.shape[0],text_with_lines_eroded.shape[1]) + + + text_with_lines_y=text_with_lines.sum(axis=0) + text_with_lines_y_eroded=text_with_lines_eroded.sum(axis=0) + + thickness_along_y_percent=text_with_lines_y_eroded.max()/(float(text_with_lines.shape[0]))*100 + + #print(thickness_along_y_percent,'thickness_along_y_percent') + + if thickness_along_y_percent<30: + min_textline_thickness=8 + elif thickness_along_y_percent>=30 and thickness_along_y_percent<50: + min_textline_thickness=20 + else: + min_textline_thickness=40 + + + + if thickness_along_y_percent>=14: + + text_with_lines_y_rev=-1*text_with_lines_y[:] + #print(text_with_lines_y) + #print(text_with_lines_y_rev) + + + + + #plt.plot(text_with_lines_y) + #plt.show() + + + text_with_lines_y_rev=text_with_lines_y_rev-np.min(text_with_lines_y_rev) + + #plt.plot(text_with_lines_y_rev) + #plt.show() + sigma_gaus=1 + region_sum_0= gaussian_filter1d(text_with_lines_y, sigma_gaus) + + region_sum_0_rev=gaussian_filter1d(text_with_lines_y_rev, sigma_gaus) + + #plt.plot(region_sum_0_rev) + #plt.show() + region_sum_0_updown=region_sum_0[len(region_sum_0)::-1] + + first_nonzero=(next((i for i, x in enumerate(region_sum_0) if x), None)) + last_nonzero=(next((i for i, x in enumerate(region_sum_0_updown) if x), None)) + + + last_nonzero=len(region_sum_0)-last_nonzero + + ##img_sum_0_smooth_rev=-region_sum_0 + + + mid_point=(last_nonzero+first_nonzero)/2. + + + one_third_right=(last_nonzero-mid_point)/3.0 + one_third_left=(mid_point-first_nonzero)/3.0 + + #img_sum_0_smooth_rev=img_sum_0_smooth_rev-np.min(img_sum_0_smooth_rev) + + + + + peaks, _ = find_peaks(text_with_lines_y_rev, height=0) + + + peaks=np.array(peaks) + + + #print(region_sum_0[peaks]) + ##plt.plot(region_sum_0) + ##plt.plot(peaks,region_sum_0[peaks],'*') + ##plt.show() + #print(first_nonzero,last_nonzero,peaks) + peaks=peaks[(peaks>first_nonzero) & ((peaksmid_point] + peaks_left=peaks[peaks(mid_point+one_third_right)] + peaks_left=peaks[peaks<(mid_point-one_third_left)] + + + try: + point_right=np.min(peaks_right) + except: + point_right=last_nonzero + + + try: + point_left=np.max(peaks_left) + except: + point_left=first_nonzero + + + + + #print(point_left,point_right) + #print(text_regions.shape) + if point_right>=mask_marginals.shape[1]: + point_right=mask_marginals.shape[1]-1 + + try: + mask_marginals[:,point_left:point_right]=1 + except: + mask_marginals[:,:]=1 + + #print(mask_marginals.shape,point_left,point_right,'nadosh') + mask_marginals_rotated=rotate_image(mask_marginals,-slope_deskew) + + #print(mask_marginals_rotated.shape,'nadosh') + mask_marginals_rotated_sum=mask_marginals_rotated.sum(axis=0) + + mask_marginals_rotated_sum[mask_marginals_rotated_sum!=0]=1 + index_x=np.array(range(len(mask_marginals_rotated_sum)))+1 + + index_x_interest=index_x[mask_marginals_rotated_sum==1] + + min_point_of_left_marginal=np.min(index_x_interest)-16 + max_point_of_right_marginal=np.max(index_x_interest)+16 + + if min_point_of_left_marginal<0: + min_point_of_left_marginal=0 + if max_point_of_right_marginal>=text_regions.shape[1]: + max_point_of_right_marginal=text_regions.shape[1]-1 + + + #print(np.min(index_x_interest) ,np.max(index_x_interest),'minmaxnew') + #print(mask_marginals_rotated.shape,text_regions.shape,'mask_marginals_rotated') + #plt.imshow(mask_marginals) + #plt.show() + + #plt.imshow(mask_marginals_rotated) + #plt.show() + + text_regions[(mask_marginals_rotated[:,:]!=1) & (text_regions[:,:]==1)]=4 + + #plt.imshow(text_regions) + #plt.show() + + pixel_img=4 + min_area_text=0.00001 + polygons_of_marginals=return_contours_of_interested_region(text_regions,pixel_img,min_area_text) + + cx_text_only,cy_text_only ,x_min_text_only,x_max_text_only, y_min_text_only ,y_max_text_only,y_cor_x_min_main=find_new_features_of_contoures(polygons_of_marginals) + + text_regions[(text_regions[:,:]==4)]=1 + + marginlas_should_be_main_text=[] + + x_min_marginals_left=[] + x_min_marginals_right=[] + + for i in range(len(cx_text_only)): + + x_width_mar=abs(x_min_text_only[i]-x_max_text_only[i]) + y_height_mar=abs(y_min_text_only[i]-y_max_text_only[i]) + #print(x_width_mar,y_height_mar,y_height_mar/x_width_mar,'y_height_mar') + if x_width_mar>16 and y_height_mar/x_width_mar<18: + marginlas_should_be_main_text.append(polygons_of_marginals[i]) + if x_min_text_only[i]<(mid_point-one_third_left): + x_min_marginals_left_new=x_min_text_only[i] + if len(x_min_marginals_left)==0: + x_min_marginals_left.append(x_min_marginals_left_new) + else: + x_min_marginals_left[0]=min(x_min_marginals_left[0],x_min_marginals_left_new) + else: + x_min_marginals_right_new=x_min_text_only[i] + if len(x_min_marginals_right)==0: + x_min_marginals_right.append(x_min_marginals_right_new) + else: + x_min_marginals_right[0]=min(x_min_marginals_right[0],x_min_marginals_right_new) + + if len(x_min_marginals_left)==0: + x_min_marginals_left=[0] + if len(x_min_marginals_right)==0: + x_min_marginals_right=[text_regions.shape[1]-1] + + + + + #print(x_min_marginals_left[0],x_min_marginals_right[0],'margo') + + #print(marginlas_should_be_main_text,'marginlas_should_be_main_text') + text_regions=cv2.fillPoly(text_regions, pts =marginlas_should_be_main_text, color=(4,4)) + + #print(np.unique(text_regions)) + + #text_regions[:,:int(x_min_marginals_left[0])][text_regions[:,:int(x_min_marginals_left[0])]==1]=0 + #text_regions[:,int(x_min_marginals_right[0]):][text_regions[:,int(x_min_marginals_right[0]):]==1]=0 + + text_regions[:,:int(min_point_of_left_marginal)][text_regions[:,:int(min_point_of_left_marginal)]==1]=0 + text_regions[:,int(max_point_of_right_marginal):][text_regions[:,int(max_point_of_right_marginal):]==1]=0 + + ###text_regions[:,0:point_left][text_regions[:,0:point_left]==1]=4 + + ###text_regions[:,point_right:][ text_regions[:,point_right:]==1]=4 + #plt.plot(region_sum_0) + #plt.plot(peaks,region_sum_0[peaks],'*') + #plt.show() + + + #plt.imshow(text_regions) + #plt.show() + + #sys.exit() + else: + pass + return text_regions diff --git a/sbb_newspapers_org_image/utils/resize.py b/sbb_newspapers_org_image/utils/resize.py new file mode 100644 index 0000000..fdc49ec --- /dev/null +++ b/sbb_newspapers_org_image/utils/resize.py @@ -0,0 +1,4 @@ +import cv2 + +def resize_image(img_in, input_height, input_width): + return cv2.resize(img_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) diff --git a/sbb_newspapers_org_image/utils/rotate.py b/sbb_newspapers_org_image/utils/rotate.py new file mode 100644 index 0000000..9cd7981 --- /dev/null +++ b/sbb_newspapers_org_image/utils/rotate.py @@ -0,0 +1,85 @@ +import math + +import imutils +import cv2 + +def rotatedRectWithMaxArea(w, h, angle): + if w <= 0 or h <= 0: + return 0, 0 + + width_is_longer = w >= h + side_long, side_short = (w, h) if width_is_longer else (h, w) + + # since the solutions for angle, -angle and 180-angle are all the same, + # if suffices to look at the first quadrant and the absolute values of sin,cos: + sin_a, cos_a = abs(math.sin(angle)), abs(math.cos(angle)) + if side_short <= 2.0 * sin_a * cos_a * side_long or abs(sin_a - cos_a) < 1e-10: + # half constrained case: two crop corners touch the longer side, + # the other two corners are on the mid-line parallel to the longer line + x = 0.5 * side_short + wr, hr = (x / sin_a, x / cos_a) if width_is_longer else (x / cos_a, x / sin_a) + else: + # fully constrained case: crop touches all 4 sides + cos_2a = cos_a * cos_a - sin_a * sin_a + wr, hr = (w * cos_a - h * sin_a) / cos_2a, (h * cos_a - w * sin_a) / cos_2a + + return wr, hr + +def rotate_max_area_new(image, rotated, angle): + wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle)) + h, w, _ = rotated.shape + y1 = h // 2 - int(hr / 2) + y2 = y1 + int(hr) + x1 = w // 2 - int(wr / 2) + x2 = x1 + int(wr) + return rotated[y1:y2, x1:x2] + +def rotation_image_new(img, thetha): + rotated = imutils.rotate(img, thetha) + return rotate_max_area_new(img, rotated, thetha) + +def rotate_image(img_patch, slope): + (h, w) = img_patch.shape[:2] + center = (w // 2, h // 2) + M = cv2.getRotationMatrix2D(center, slope, 1.0) + return cv2.warpAffine(img_patch, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) + +def rotyate_image_different( img, slope): + # img = cv2.imread('images/input.jpg') + num_rows, num_cols = img.shape[:2] + + rotation_matrix = cv2.getRotationMatrix2D((num_cols / 2, num_rows / 2), slope, 1) + img_rotation = cv2.warpAffine(img, rotation_matrix, (num_cols, num_rows)) + return img_rotation + +def rotate_max_area(image, rotated, rotated_textline, rotated_layout, angle): + wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle)) + h, w, _ = rotated.shape + y1 = h // 2 - int(hr / 2) + y2 = y1 + int(hr) + x1 = w // 2 - int(wr / 2) + x2 = x1 + int(wr) + return rotated[y1:y2, x1:x2], rotated_textline[y1:y2, x1:x2], rotated_layout[y1:y2, x1:x2] + +def rotation_not_90_func(img, textline, text_regions_p_1, thetha): + rotated = imutils.rotate(img, thetha) + rotated_textline = imutils.rotate(textline, thetha) + rotated_layout = imutils.rotate(text_regions_p_1, thetha) + return rotate_max_area(img, rotated, rotated_textline, rotated_layout, thetha) + +def rotation_not_90_func_full_layout(img, textline, text_regions_p_1, text_regions_p_fully, thetha): + rotated = imutils.rotate(img, thetha) + rotated_textline = imutils.rotate(textline, thetha) + rotated_layout = imutils.rotate(text_regions_p_1, thetha) + rotated_layout_full = imutils.rotate(text_regions_p_fully, thetha) + return rotate_max_area_full_layout(img, rotated, rotated_textline, rotated_layout, rotated_layout_full, thetha) + +def rotate_max_area_full_layout(image, rotated, rotated_textline, rotated_layout, rotated_layout_full, angle): + wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle)) + h, w, _ = rotated.shape + y1 = h // 2 - int(hr / 2) + y2 = y1 + int(hr) + x1 = w // 2 - int(wr / 2) + x2 = x1 + int(wr) + return rotated[y1:y2, x1:x2], rotated_textline[y1:y2, x1:x2], rotated_layout[y1:y2, x1:x2], rotated_layout_full[y1:y2, x1:x2] + diff --git a/sbb_newspapers_org_image/utils/separate_lines.py b/sbb_newspapers_org_image/utils/separate_lines.py new file mode 100644 index 0000000..740495a --- /dev/null +++ b/sbb_newspapers_org_image/utils/separate_lines.py @@ -0,0 +1,1752 @@ +import matplotlib.pyplot as plt +import numpy as np +import cv2 +from scipy.signal import find_peaks +from scipy.ndimage import gaussian_filter1d + +from .rotate import rotate_image +from .contour import ( + return_parent_contours, + filter_contours_area_of_image_tables, + return_contours_of_image, + filter_contours_area_of_image +) +from .is_nan import isNaN +from .utils import ( + boosting_headers_by_longshot_region_segmentation, + crop_image_inside_box, + find_features_of_lines, + find_num_col, + find_num_col_by_vertical_lines, + find_num_col_deskew, + find_num_col_only_image, + isNaN, + otsu_copy, + otsu_copy_binary, + return_hor_spliter_by_index_for_without_verticals, + delete_seperator_around, + return_regions_without_seperators, + put_drop_out_from_only_drop_model, + putt_bb_of_drop_capitals_of_model_in_patches_in_layout, + check_any_text_region_in_model_one_is_main_or_header, + small_textlines_to_parent_adherence2, + order_and_id_of_texts, + order_of_regions, + implent_law_head_main_not_parallel, + return_hor_spliter_by_index, + combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new, + return_points_with_boundies, + find_number_of_columns_in_document, + return_boxes_of_images_by_order_of_reading_new, +) + +def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): + (h, w) = img_patch.shape[:2] + center = (w // 2, h // 2) + M = cv2.getRotationMatrix2D(center, -thetha, 1.0) + x_d = M[0, 2] + y_d = M[1, 2] + + thetha = thetha / 180.0 * np.pi + rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + + x_cont = contour_text_interest[:, 0, 0] + y_cont = contour_text_interest[:, 0, 1] + x_cont = x_cont - np.min(x_cont) + y_cont = y_cont - np.min(y_cont) + + x_min_cont = 0 + x_max_cont = img_patch.shape[1] + y_min_cont = 0 + y_max_cont = img_patch.shape[0] + + xv = np.linspace(x_min_cont, x_max_cont, 1000) + + textline_patch_sum_along_width = img_patch.sum(axis=axis) + + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + + y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] + y_padded = np.zeros(len(y) + 40) + y_padded[20 : len(y) + 20] = y + x = np.array(range(len(y))) + + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + + if 1 > 0: + + try: + + y_padded_smoothed_e = gaussian_filter1d(y_padded, 2) + y_padded_up_to_down_e = -y_padded + np.max(y_padded) + y_padded_up_to_down_padded_e = np.zeros(len(y_padded_up_to_down_e) + 40) + y_padded_up_to_down_padded_e[20 : len(y_padded_up_to_down_e) + 20] = y_padded_up_to_down_e + y_padded_up_to_down_padded_e = gaussian_filter1d(y_padded_up_to_down_padded_e, 2) + + peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) + peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) + neg_peaks_max = np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + + arg_neg_must_be_deleted = np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) + + arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] + + peaks_new = peaks_e[:] + peaks_neg_new = peaks_neg_e[:] + + clusters_to_be_deleted = [] + if len(arg_diff_cluster) > 0: + + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) + for i in range(len(arg_diff_cluster) - 1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + + if len(clusters_to_be_deleted) > 0: + peaks_new_extra = [] + for m in range(len(clusters_to_be_deleted)): + min_cluster = np.min(peaks_e[clusters_to_be_deleted[m]]) + max_cluster = np.max(peaks_e[clusters_to_be_deleted[m]]) + peaks_new_extra.append(int((min_cluster + max_cluster) / 2.0)) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1] - 1]] + peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1]]] + + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg_e[clusters_to_be_deleted[m][m1]]] + peaks_new_tot = [] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot = np.sort(peaks_new_tot) + + else: + peaks_new_tot = peaks_e[:] + + textline_con, hierachy = return_contours_of_image(img_patch) + textline_con_fil = filter_contours_area_of_image(img_patch, textline_con, hierachy, max_area=1, min_area=0.0008) + y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + + sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) + # print(sigma_gaus,'sigma_gaus') + except: + sigma_gaus = 12 + if sigma_gaus < 3: + sigma_gaus = 3 + # print(sigma_gaus,'sigma') + + y_padded_smoothed = gaussian_filter1d(y_padded, sigma_gaus) + y_padded_up_to_down = -y_padded + np.max(y_padded) + y_padded_up_to_down_padded = np.zeros(len(y_padded_up_to_down) + 40) + y_padded_up_to_down_padded[20 : len(y_padded_up_to_down) + 20] = y_padded_up_to_down + y_padded_up_to_down_padded = gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) + + peaks, _ = find_peaks(y_padded_smoothed, height=0) + peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) + + return x, y, x_d, y_d, xv, x_min_cont, y_min_cont, x_max_cont, y_max_cont, first_nonzero, y_padded_up_to_down_padded, y_padded_smoothed, peaks, peaks_neg, rotation_matrix + +def seperate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): + + contour_text_interest_copy = contour_text_interest.copy() + x, y, x_d, y_d, xv, x_min_cont, y_min_cont, x_max_cont, y_max_cont, first_nonzero, y_padded_up_to_down_padded, y_padded_smoothed, peaks, peaks_neg, rotation_matrix = dedup_separate_lines(img_patch, contour_text_interest, thetha, 1) + + try: + neg_peaks_max = np.max(y_padded_smoothed[peaks]) + + arg_neg_must_be_deleted = np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.42] + + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) + + arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] + except: + arg_neg_must_be_deleted = [] + arg_diff_cluster = [] + + try: + peaks_new = peaks[:] + peaks_neg_new = peaks_neg[:] + clusters_to_be_deleted = [] + + if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) + for i in range(len(arg_diff_cluster) - 1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) + + if len(arg_neg_must_be_deleted) == 1: + clusters_to_be_deleted.append(arg_neg_must_be_deleted) + + if len(clusters_to_be_deleted) > 0: + peaks_new_extra = [] + for m in range(len(clusters_to_be_deleted)): + min_cluster = np.min(peaks[clusters_to_be_deleted[m]]) + max_cluster = np.max(peaks[clusters_to_be_deleted[m]]) + peaks_new_extra.append(int((min_cluster + max_cluster) / 2.0)) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1] - 1]] + peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1]]] + + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg[clusters_to_be_deleted[m][m1]]] + peaks_new_tot = [] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot = np.sort(peaks_new_tot) + + ##plt.plot(y_padded_up_to_down_padded) + ##plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') + ##plt.show() + + ##plt.plot(y_padded_up_to_down_padded) + ##plt.plot(peaks_neg_new,y_padded_up_to_down_padded[peaks_neg_new],'*') + ##plt.show() + + ##plt.plot(y_padded_smoothed) + ##plt.plot(peaks,y_padded_smoothed[peaks],'*') + ##plt.show() + + ##plt.plot(y_padded_smoothed) + ##plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') + ##plt.show() + + peaks = peaks_new_tot[:] + peaks_neg = peaks_neg_new[:] + + else: + peaks_new_tot = peaks[:] + peaks = peaks_new_tot[:] + peaks_neg = peaks_neg_new[:] + except: + pass + + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + peaks_values = y_padded_smoothed[peaks] + + peaks_neg = peaks_neg - 20 - 20 + peaks = peaks - 20 + + for jj in range(len(peaks_neg)): + if peaks_neg[jj] > len(x) - 1: + peaks_neg[jj] = len(x) - 1 + + for jj in range(len(peaks)): + if peaks[jj] > len(x) - 1: + peaks[jj] = len(x) - 1 + + textline_boxes = [] + textline_boxes_rot = [] + + if len(peaks_neg) == len(peaks) + 1 and len(peaks) >= 3: + for jj in range(len(peaks)): + + if jj == (len(peaks) - 1): + dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) + dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) + + if peaks_values[jj] > mean_value_of_peaks - std_value_of_peaks / 2.0: + point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = y_max_cont - 1 ##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + else: + point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = y_max_cont - 1 ##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + + point_down_narrow = peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + else: + dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) + dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) + + if peaks_values[jj] > mean_value_of_peaks - std_value_of_peaks / 2.0: + point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + else: + point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + + point_down_narrow = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + + if point_down_narrow >= img_patch.shape[0]: + point_down_narrow = img_patch.shape[0] - 2 + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) + x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) + + p1 = np.dot(rotation_matrix, [int(x_min), int(point_up)]) + p2 = np.dot(rotation_matrix, [int(x_max), int(point_up)]) + p3 = np.dot(rotation_matrix, [int(x_max), int(point_down)]) + p4 = np.dot(rotation_matrix, [int(x_min), int(point_down)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + x_min_rot1 = x_min_rot1 - x_help + x_max_rot2 = x_max_rot2 - x_help + x_max_rot3 = x_max_rot3 - x_help + x_min_rot4 = x_min_rot4 - x_help + + point_up_rot1 = point_up_rot1 - y_help + point_up_rot2 = point_up_rot2 - y_help + point_down_rot3 = point_down_rot3 - y_help + point_down_rot4 = point_down_rot4 - y_help + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + + elif len(peaks) < 1: + pass + + elif len(peaks) == 1: + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[0] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) + x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) + # x_min = x_min_cont + # x_max = x_max_cont + + y_min = y_min_cont + y_max = y_max_cont + + p1 = np.dot(rotation_matrix, [int(x_min), int(y_min)]) + p2 = np.dot(rotation_matrix, [int(x_max), int(y_min)]) + p3 = np.dot(rotation_matrix, [int(x_max), int(y_max)]) + p4 = np.dot(rotation_matrix, [int(x_min), int(y_max)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + x_min_rot1 = x_min_rot1 - x_help + x_max_rot2 = x_max_rot2 - x_help + x_max_rot3 = x_max_rot3 - x_help + x_min_rot4 = x_min_rot4 - x_help + + point_up_rot1 = point_up_rot1 - y_help + point_up_rot2 = point_up_rot2 - y_help + point_down_rot3 = point_down_rot3 - y_help + point_down_rot4 = point_down_rot4 - y_help + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(y_min)], [int(x_max), int(y_min)], [int(x_max), int(y_max)], [int(x_min), int(y_max)]])) + + elif len(peaks) == 2: + dis_to_next = np.abs(peaks[1] - peaks[0]) + for jj in range(len(peaks)): + if jj == 0: + point_up = 0 # peaks[jj] + first_nonzero - int(1. / 1.7 * dis_to_next) + if point_up < 0: + point_up = 1 + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.8 * dis_to_next) + elif jj == 1: + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.8 * dis_to_next) + if point_down >= img_patch.shape[0]: + point_down = img_patch.shape[0] - 2 + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.8 * dis_to_next) + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) + x_max = np.max(xvinside) + + p1 = np.dot(rotation_matrix, [int(x_min), int(point_up)]) + p2 = np.dot(rotation_matrix, [int(x_max), int(point_up)]) + p3 = np.dot(rotation_matrix, [int(x_max), int(point_down)]) + p4 = np.dot(rotation_matrix, [int(x_min), int(point_down)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + x_min_rot1 = x_min_rot1 - x_help + x_max_rot2 = x_max_rot2 - x_help + x_max_rot3 = x_max_rot3 - x_help + x_min_rot4 = x_min_rot4 - x_help + + point_up_rot1 = point_up_rot1 - y_help + point_up_rot2 = point_up_rot2 - y_help + point_down_rot3 = point_down_rot3 - y_help + point_down_rot4 = point_down_rot4 - y_help + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + else: + for jj in range(len(peaks)): + + if jj == 0: + dis_to_next = peaks[jj + 1] - peaks[jj] + # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next) + if point_up < 0: + point_up = 1 + # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.9 * dis_to_next) + elif jj == len(peaks) - 1: + dis_to_next = peaks[jj] - peaks[jj - 1] + # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.7 * dis_to_next) + if point_down >= img_patch.shape[0]: + point_down = img_patch.shape[0] - 2 + # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next) + else: + dis_to_next_down = peaks[jj + 1] - peaks[jj] + dis_to_next_up = peaks[jj] - peaks[jj - 1] + + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next_up) + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.9 * dis_to_next_down) + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) + x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) + + p1 = np.dot(rotation_matrix, [int(x_min), int(point_up)]) + p2 = np.dot(rotation_matrix, [int(x_max), int(point_up)]) + p3 = np.dot(rotation_matrix, [int(x_max), int(point_down)]) + p4 = np.dot(rotation_matrix, [int(x_min), int(point_down)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + x_min_rot1 = x_min_rot1 - x_help + x_max_rot2 = x_max_rot2 - x_help + x_max_rot3 = x_max_rot3 - x_help + x_min_rot4 = x_min_rot4 - x_help + + point_up_rot1 = point_up_rot1 - y_help + point_up_rot2 = point_up_rot2 - y_help + point_down_rot3 = point_down_rot3 - y_help + point_down_rot4 = point_down_rot4 - y_help + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + + return peaks, textline_boxes_rot + +def seperate_lines_vertical(img_patch, contour_text_interest, thetha): + + thetha = thetha + 90 + contour_text_interest_copy = contour_text_interest.copy() + x, y, x_d, y_d, xv, x_min_cont, y_min_cont, x_max_cont, y_max_cont, first_nonzero, y_padded_up_to_down_padded, y_padded_smoothed, peaks, peaks_neg, rotation_matrix = dedup_separate_lines(img_patch, contour_text_interest, thetha, 0) + + + # plt.plot(y_padded_up_to_down_padded) + # plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') + # plt.title('negs') + # plt.show() + + # plt.plot(y_padded_smoothed) + # plt.plot(peaks,y_padded_smoothed[peaks],'*') + # plt.title('poss') + # plt.show() + + neg_peaks_max = np.max(y_padded_up_to_down_padded[peaks_neg]) + + arg_neg_must_be_deleted = np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.42] + + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) + + arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] + + peaks_new = peaks[:] + peaks_neg_new = peaks_neg[:] + clusters_to_be_deleted = [] + + if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) + for i in range(len(arg_diff_cluster) - 1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) + + if len(arg_neg_must_be_deleted) == 1: + clusters_to_be_deleted.append(arg_neg_must_be_deleted) + + if len(clusters_to_be_deleted) > 0: + peaks_new_extra = [] + for m in range(len(clusters_to_be_deleted)): + min_cluster = np.min(peaks[clusters_to_be_deleted[m]]) + max_cluster = np.max(peaks[clusters_to_be_deleted[m]]) + peaks_new_extra.append(int((min_cluster + max_cluster) / 2.0)) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1] - 1]] + peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1]]] + + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg[clusters_to_be_deleted[m][m1]]] + peaks_new_tot = [] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot = np.sort(peaks_new_tot) + + peaks = peaks_new_tot[:] + peaks_neg = peaks_neg_new[:] + + else: + peaks_new_tot = peaks[:] + peaks = peaks_new_tot[:] + peaks_neg = peaks_neg_new[:] + + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + peaks_values = y_padded_smoothed[peaks] + + peaks_neg = peaks_neg - 20 - 20 + peaks = peaks - 20 + + for jj in range(len(peaks_neg)): + if peaks_neg[jj] > len(x) - 1: + peaks_neg[jj] = len(x) - 1 + + for jj in range(len(peaks)): + if peaks[jj] > len(x) - 1: + peaks[jj] = len(x) - 1 + + textline_boxes = [] + textline_boxes_rot = [] + + if len(peaks_neg) == len(peaks) + 1 and len(peaks) >= 3: + # print('11') + for jj in range(len(peaks)): + + if jj == (len(peaks) - 1): + dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) + dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) + + if peaks_values[jj] > mean_value_of_peaks - std_value_of_peaks / 2.0: + point_up = peaks[jj] + first_nonzero - int(1.3 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = x_max_cont - 1 ##peaks[jj] + first_nonzero + int(1.3 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + else: + point_up = peaks[jj] + first_nonzero - int(1.4 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = x_max_cont - 1 ##peaks[jj] + first_nonzero + int(1.6 * dis_to_next_down) #point_up# np.max(y_cont)#peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + + point_down_narrow = peaks[jj] + first_nonzero + int(1.4 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + else: + dis_to_next_up = abs(peaks[jj] - peaks_neg[jj]) + dis_to_next_down = abs(peaks[jj] - peaks_neg[jj + 1]) + + if peaks_values[jj] > mean_value_of_peaks - std_value_of_peaks / 2.0: + point_up = peaks[jj] + first_nonzero - int(1.1 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + else: + point_up = peaks[jj] + first_nonzero - int(1.23 * dis_to_next_up) ##+int(dis_to_next_up*1./4.0) + point_down = peaks[jj] + first_nonzero + int(1.33 * dis_to_next_down) ###-int(dis_to_next_down*1./4.0) + + point_down_narrow = peaks[jj] + first_nonzero + int(1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + + if point_down_narrow >= img_patch.shape[0]: + point_down_narrow = img_patch.shape[0] - 2 + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) + x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) + + p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) + p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) + p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) + p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + + elif len(peaks) < 1: + pass + + elif len(peaks) == 1: + x_min = x_min_cont + x_max = x_max_cont + + y_min = y_min_cont + y_max = y_max_cont + + p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) + p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) + p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) + p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(y_min)], [int(x_max), int(y_min)], [int(x_max), int(y_max)], [int(x_min), int(y_max)]])) + + elif len(peaks) == 2: + dis_to_next = np.abs(peaks[1] - peaks[0]) + for jj in range(len(peaks)): + if jj == 0: + point_up = 0 # peaks[jj] + first_nonzero - int(1. / 1.7 * dis_to_next) + if point_up < 0: + point_up = 1 + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.8 * dis_to_next) + elif jj == 1: + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.8 * dis_to_next) + if point_down >= img_patch.shape[0]: + point_down = img_patch.shape[0] - 2 + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.8 * dis_to_next) + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) + x_max = np.max(xvinside) + + p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) + p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) + p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) + p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + else: + for jj in range(len(peaks)): + + if jj == 0: + dis_to_next = peaks[jj + 1] - peaks[jj] + # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next) + if point_up < 0: + point_up = 1 + # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.9 * dis_to_next) + elif jj == len(peaks) - 1: + dis_to_next = peaks[jj] - peaks[jj - 1] + # point_down=peaks[jj]+first_nonzero+int(1./3*dis_to_next) + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.7 * dis_to_next) + if point_down >= img_patch.shape[0]: + point_down = img_patch.shape[0] - 2 + # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next) + else: + dis_to_next_down = peaks[jj + 1] - peaks[jj] + dis_to_next_up = peaks[jj] - peaks[jj - 1] + + point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next_up) + point_down = peaks[jj] + first_nonzero + int(1.0 / 1.9 * dis_to_next_down) + + distances = [cv2.pointPolygonTest(contour_text_interest_copy, (xv[mj], peaks[jj] + first_nonzero), True) for mj in range(len(xv))] + distances = np.array(distances) + + xvinside = xv[distances >= 0] + + if len(xvinside) == 0: + x_min = x_min_cont + x_max = x_max_cont + else: + x_min = np.min(xvinside) # max(x_min_interest,x_min_cont) + x_max = np.max(xvinside) # min(x_max_interest,x_max_cont) + + p1 = np.dot(rotation_matrix, [int(point_up), int(y_min_cont)]) + p2 = np.dot(rotation_matrix, [int(point_down), int(y_min_cont)]) + p3 = np.dot(rotation_matrix, [int(point_down), int(y_max_cont)]) + p4 = np.dot(rotation_matrix, [int(point_up), int(y_max_cont)]) + + x_min_rot1, point_up_rot1 = p1[0] + x_d, p1[1] + y_d + x_max_rot2, point_up_rot2 = p2[0] + x_d, p2[1] + y_d + x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d + x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1 < 0: + x_min_rot1 = 0 + if x_min_rot4 < 0: + x_min_rot4 = 0 + if point_up_rot1 < 0: + point_up_rot1 = 0 + if point_up_rot2 < 0: + point_up_rot2 = 0 + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + + return peaks, textline_boxes_rot + +def seperate_lines_new_inside_teils2(img_patch, thetha): + + (h, w) = img_patch.shape[:2] + center = (w // 2, h // 2) + M = cv2.getRotationMatrix2D(center, -thetha, 1.0) + x_d = M[0, 2] + y_d = M[1, 2] + + thetha = thetha / 180.0 * np.pi + rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + # contour_text_interest_copy = contour_text_interest.copy() + + # x_cont = contour_text_interest[:, 0, 0] + # y_cont = contour_text_interest[:, 0, 1] + # x_cont = x_cont - np.min(x_cont) + # y_cont = y_cont - np.min(y_cont) + + x_min_cont = 0 + x_max_cont = img_patch.shape[1] + y_min_cont = 0 + y_max_cont = img_patch.shape[0] + + xv = np.linspace(x_min_cont, x_max_cont, 1000) + + textline_patch_sum_along_width = img_patch.sum(axis=1) + + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + + y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] + y_padded = np.zeros(len(y) + 40) + y_padded[20 : len(y) + 20] = y + x = np.array(range(len(y))) + + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + if 1 > 0: + + try: + + y_padded_smoothed_e = gaussian_filter1d(y_padded, 2) + y_padded_up_to_down_e = -y_padded + np.max(y_padded) + y_padded_up_to_down_padded_e = np.zeros(len(y_padded_up_to_down_e) + 40) + y_padded_up_to_down_padded_e[20 : len(y_padded_up_to_down_e) + 20] = y_padded_up_to_down_e + y_padded_up_to_down_padded_e = gaussian_filter1d(y_padded_up_to_down_padded_e, 2) + + peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) + peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) + neg_peaks_max = np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + + arg_neg_must_be_deleted = np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) + + arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] + + peaks_new = peaks_e[:] + peaks_neg_new = peaks_neg_e[:] + + clusters_to_be_deleted = [] + if len(arg_diff_cluster) > 0: + + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) + for i in range(len(arg_diff_cluster) - 1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + + if len(clusters_to_be_deleted) > 0: + peaks_new_extra = [] + for m in range(len(clusters_to_be_deleted)): + min_cluster = np.min(peaks_e[clusters_to_be_deleted[m]]) + max_cluster = np.max(peaks_e[clusters_to_be_deleted[m]]) + peaks_new_extra.append(int((min_cluster + max_cluster) / 2.0)) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1] - 1]] + peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1]]] + + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg_e[clusters_to_be_deleted[m][m1]]] + peaks_new_tot = [] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot = np.sort(peaks_new_tot) + + else: + peaks_new_tot = peaks_e[:] + + textline_con, hierachy = return_contours_of_image(img_patch) + textline_con_fil = filter_contours_area_of_image(img_patch, textline_con, hierachy, max_area=1, min_area=0.0008) + y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + + sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) + # print(sigma_gaus,'sigma_gaus') + except: + sigma_gaus = 12 + if sigma_gaus < 3: + sigma_gaus = 3 + # print(sigma_gaus,'sigma') + + y_padded_smoothed = gaussian_filter1d(y_padded, sigma_gaus) + y_padded_up_to_down = -y_padded + np.max(y_padded) + y_padded_up_to_down_padded = np.zeros(len(y_padded_up_to_down) + 40) + y_padded_up_to_down_padded[20 : len(y_padded_up_to_down) + 20] = y_padded_up_to_down + y_padded_up_to_down_padded = gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) + + peaks, _ = find_peaks(y_padded_smoothed, height=0) + peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) + + peaks_new = peaks[:] + peaks_neg_new = peaks_neg[:] + + try: + neg_peaks_max = np.max(y_padded_smoothed[peaks]) + + arg_neg_must_be_deleted = np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.24] + + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) + + arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] + + clusters_to_be_deleted = [] + + if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) + for i in range(len(arg_diff_cluster) - 1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) + + if len(arg_neg_must_be_deleted) == 1: + clusters_to_be_deleted.append(arg_neg_must_be_deleted) + + if len(clusters_to_be_deleted) > 0: + peaks_new_extra = [] + for m in range(len(clusters_to_be_deleted)): + min_cluster = np.min(peaks[clusters_to_be_deleted[m]]) + max_cluster = np.max(peaks[clusters_to_be_deleted[m]]) + peaks_new_extra.append(int((min_cluster + max_cluster) / 2.0)) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1] - 1]] + peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1]]] + + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg[clusters_to_be_deleted[m][m1]]] + peaks_new_tot = [] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot = np.sort(peaks_new_tot) + + # plt.plot(y_padded_up_to_down_padded) + # plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') + # plt.show() + + # plt.plot(y_padded_up_to_down_padded) + # plt.plot(peaks_neg_new,y_padded_up_to_down_padded[peaks_neg_new],'*') + # plt.show() + + # plt.plot(y_padded_smoothed) + # plt.plot(peaks,y_padded_smoothed[peaks],'*') + # plt.show() + + # plt.plot(y_padded_smoothed) + # plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') + # plt.show() + + peaks = peaks_new_tot[:] + peaks_neg = peaks_neg_new[:] + except: + pass + + else: + peaks_new_tot = peaks[:] + peaks = peaks_new_tot[:] + peaks_neg = peaks_neg_new[:] + + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) + peaks_values = y_padded_smoothed[peaks] + + ###peaks_neg = peaks_neg - 20 - 20 + ###peaks = peaks - 20 + peaks_neg_true = peaks_neg[:] + peaks_pos_true = peaks[:] + + if len(peaks_neg_true) > 0: + peaks_neg_true = np.array(peaks_neg_true) + + peaks_neg_true = peaks_neg_true - 20 - 20 + + # print(peaks_neg_true) + for i in range(len(peaks_neg_true)): + img_patch[peaks_neg_true[i] - 6 : peaks_neg_true[i] + 6, :] = 0 + + else: + pass + + if len(peaks_pos_true) > 0: + peaks_pos_true = np.array(peaks_pos_true) + peaks_pos_true = peaks_pos_true - 20 + + for i in range(len(peaks_pos_true)): + ##img_patch[peaks_pos_true[i]-8:peaks_pos_true[i]+8,:]=1 + img_patch[peaks_pos_true[i] - 6 : peaks_pos_true[i] + 6, :] = 1 + else: + pass + kernel = np.ones((5, 5), np.uint8) + + # img_patch = cv2.erode(img_patch,kernel,iterations = 3) + #######################img_patch = cv2.erode(img_patch,kernel,iterations = 2) + img_patch = cv2.erode(img_patch, kernel, iterations=1) + return img_patch + +def seperate_lines_new_inside_teils(img_path, thetha): + (h, w) = img_path.shape[:2] + center = (w // 2, h // 2) + M = cv2.getRotationMatrix2D(center, -thetha, 1.0) + x_d = M[0, 2] + y_d = M[1, 2] + + thetha = thetha / 180.0 * np.pi + rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) + + x_min_cont = 0 + x_max_cont = img_path.shape[1] + y_min_cont = 0 + y_max_cont = img_path.shape[0] + + xv = np.linspace(x_min_cont, x_max_cont, 1000) + + mada_n = img_path.sum(axis=1) + + ##plt.plot(mada_n) + ##plt.show() + + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) + + y = mada_n[:] # [first_nonzero:last_nonzero] + y_help = np.zeros(len(y) + 40) + y_help[20 : len(y) + 20] = y + x = np.array(range(len(y))) + + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + if len(peaks_real) <= 2 and len(peaks_real) > 1: + sigma_gaus = 10 + else: + sigma_gaus = 5 + + z = gaussian_filter1d(y_help, sigma_gaus) + zneg_rev = -y_help + np.max(y_help) + zneg = np.zeros(len(zneg_rev) + 40) + zneg[20 : len(zneg_rev) + 20] = zneg_rev + zneg = gaussian_filter1d(zneg, sigma_gaus) + + peaks, _ = find_peaks(z, height=0) + peaks_neg, _ = find_peaks(zneg, height=0) + + for nn in range(len(peaks_neg)): + if peaks_neg[nn] > len(z) - 1: + peaks_neg[nn] = len(z) - 1 + if peaks_neg[nn] < 0: + peaks_neg[nn] = 0 + + diff_peaks = np.abs(np.diff(peaks_neg)) + + cut_off = 20 + peaks_neg_true = [] + forest = [] + + for i in range(len(peaks_neg)): + if i == 0: + forest.append(peaks_neg[i]) + if i < (len(peaks_neg) - 1): + if diff_peaks[i] <= cut_off: + forest.append(peaks_neg[i + 1]) + if diff_peaks[i] > cut_off: + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmin(z[forest])]): + peaks_neg_true.append(forest[np.argmin(z[forest])]) + forest = [] + forest.append(peaks_neg[i + 1]) + if i == (len(peaks_neg) - 1): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmin(z[forest])]): + peaks_neg_true.append(forest[np.argmin(z[forest])]) + + diff_peaks_pos = np.abs(np.diff(peaks)) + + cut_off = 20 + peaks_pos_true = [] + forest = [] + + for i in range(len(peaks)): + if i == 0: + forest.append(peaks[i]) + if i < (len(peaks) - 1): + if diff_peaks_pos[i] <= cut_off: + forest.append(peaks[i + 1]) + if diff_peaks_pos[i] > cut_off: + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmax(z[forest])]): + peaks_pos_true.append(forest[np.argmax(z[forest])]) + forest = [] + forest.append(peaks[i + 1]) + if i == (len(peaks) - 1): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmax(z[forest])]): + peaks_pos_true.append(forest[np.argmax(z[forest])]) + + # print(len(peaks_neg_true) ,len(peaks_pos_true) ,'lensss') + + if len(peaks_neg_true) > 0: + peaks_neg_true = np.array(peaks_neg_true) + """ + #plt.figure(figsize=(40,40)) + #plt.subplot(1,2,1) + #plt.title('Textline segmentation von Textregion') + #plt.imshow(img_path) + #plt.xlabel('X') + #plt.ylabel('Y') + #plt.subplot(1,2,2) + #plt.title('Dichte entlang X') + #base = pyplot.gca().transData + #rot = transforms.Affine2D().rotate_deg(90) + #plt.plot(zneg,np.array(range(len(zneg)))) + #plt.plot(zneg[peaks_neg_true],peaks_neg_true,'*') + #plt.gca().invert_yaxis() + + #plt.xlabel('Dichte') + #plt.ylabel('Y') + ##plt.plot([0,len(y)], [grenze,grenze]) + #plt.show() + """ + peaks_neg_true = peaks_neg_true - 20 - 20 + + # print(peaks_neg_true) + for i in range(len(peaks_neg_true)): + img_path[peaks_neg_true[i] - 6 : peaks_neg_true[i] + 6, :] = 0 + + else: + pass + + if len(peaks_pos_true) > 0: + peaks_pos_true = np.array(peaks_pos_true) + peaks_pos_true = peaks_pos_true - 20 + + for i in range(len(peaks_pos_true)): + img_path[peaks_pos_true[i] - 8 : peaks_pos_true[i] + 8, :] = 1 + else: + pass + kernel = np.ones((5, 5), np.uint8) + + # img_path = cv2.erode(img_path,kernel,iterations = 3) + img_path = cv2.erode(img_path, kernel, iterations=2) + return img_path + +def seperate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_ind, add_boxes_coor_into_textlines): + kernel = np.ones((5, 5), np.uint8) + pixel = 255 + min_area = 0 + max_area = 1 + + if len(img_patch.shape) == 3: + cnts_images = (img_patch[:, :, 0] == pixel) * 1 + else: + cnts_images = (img_patch[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_imgs, hiearchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hiearchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hiearchy, max_area=max_area, min_area=min_area) + + cont_final = [] + ###print(add_boxes_coor_into_textlines,'ikki') + for i in range(len(contours_imgs)): + img_contour = np.zeros((cnts_images.shape[0], cnts_images.shape[1], 3)) + img_contour = cv2.fillPoly(img_contour, pts=[contours_imgs[i]], color=(255, 255, 255)) + + img_contour = img_contour.astype(np.uint8) + + img_contour = cv2.dilate(img_contour, kernel, iterations=4) + imgrayrot = cv2.cvtColor(img_contour, cv2.COLOR_BGR2GRAY) + _, threshrot = cv2.threshold(imgrayrot, 0, 255, 0) + contours_text_rot, _ = cv2.findContours(threshrot.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + ##contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[ + ##0] + ##contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] + ##if add_boxes_coor_into_textlines: + ##print(np.shape(contours_text_rot[0]),'sjppo') + ##contours_text_rot[0][:, 0, 0]=contours_text_rot[0][:, 0, 0] + box_ind[0] + ##contours_text_rot[0][:, 0, 1]=contours_text_rot[0][:, 0, 1] + box_ind[1] + cont_final.append(contours_text_rot[0]) + + ##print(cont_final,'nadizzzz') + return None, cont_final + + +def textline_contours_postprocessing(textline_mask, slope, contour_text_interest, box_ind, slope_first, add_boxes_coor_into_textlines=False): + + textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 + textline_mask = textline_mask.astype(np.uint8) + kernel = np.ones((5, 5), np.uint8) + textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_OPEN, kernel) + textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, kernel) + textline_mask = cv2.erode(textline_mask, kernel, iterations=2) + # textline_mask = cv2.erode(textline_mask, kernel, iterations=1) + + # print(textline_mask.shape[0]/float(textline_mask.shape[1]),'miz') + try: + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(textline_mask) + # plt.show() + + # if abs(slope)>1: + # x_help=30 + # y_help=2 + # else: + # x_help=2 + # y_help=2 + + x_help = 30 + y_help = 2 + + textline_mask_help = np.zeros((textline_mask.shape[0] + int(2 * y_help), textline_mask.shape[1] + int(2 * x_help), 3)) + textline_mask_help[y_help : y_help + textline_mask.shape[0], x_help : x_help + textline_mask.shape[1], :] = np.copy(textline_mask[:, :, :]) + + dst = rotate_image(textline_mask_help, slope) + dst = dst[:, :, 0] + dst[dst != 0] = 1 + + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(dst) + # plt.show() + + contour_text_copy = contour_text_interest.copy() + + contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[0] + contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] + + img_contour = np.zeros((box_ind[3], box_ind[2], 3)) + img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=(255, 255, 255)) + + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(img_contour) + # plt.show() + + img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), img_contour.shape[1] + int(2 * x_help), 3)) + + img_contour_help[y_help : y_help + img_contour.shape[0], x_help : x_help + img_contour.shape[1], :] = np.copy(img_contour[:, :, :]) + + img_contour_rot = rotate_image(img_contour_help, slope) + + # plt.imshow(img_contour_rot_help) + # plt.show() + + # plt.imshow(dst_help) + # plt.show() + + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(img_contour_rot_help) + # plt.show() + + # plt.imshow(dst_help) + # plt.show() + + img_contour_rot = img_contour_rot.astype(np.uint8) + # dst_help = dst_help.astype(np.uint8) + imgrayrot = cv2.cvtColor(img_contour_rot, cv2.COLOR_BGR2GRAY) + _, threshrot = cv2.threshold(imgrayrot, 0, 255, 0) + contours_text_rot, _ = cv2.findContours(threshrot.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] + ind_big_con = np.argmax(len_con_text_rot) + + # print('juzaa') + if abs(slope) > 45: + # print(add_boxes_coor_into_textlines,'avval') + _, contours_rotated_clean = seperate_lines_vertical_cont(textline_mask, contours_text_rot[ind_big_con], box_ind, slope, add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) + else: + _, contours_rotated_clean = seperate_lines(dst, contours_text_rot[ind_big_con], slope, x_help, y_help) + + except: + + contours_rotated_clean = [] + + return contours_rotated_clean + +def seperate_lines_new2(img_path, thetha, num_col, slope_region, dir_of_all, f_name): + + if num_col == 1: + num_patches = int(img_path.shape[1] / 200.0) + else: + num_patches = int(img_path.shape[1] / 140.0) + # num_patches=int(img_path.shape[1]/200.) + if num_patches == 0: + num_patches = 1 + + img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[15]+dis_down ,:] + + # plt.imshow(img_patch_ineterst) + # plt.show() + + length_x = int(img_path.shape[1] / float(num_patches)) + # margin = int(0.04 * length_x) just recently this was changed because it break lines into 2 + margin = int(0.04 * length_x) + # print(margin,'margin') + # if margin<=4: + # margin = int(0.08 * length_x) + + # margin=0 + + width_mid = length_x - 2 * margin + + nxf = img_path.shape[1] / float(width_mid) + + if nxf > int(nxf): + nxf = int(nxf) + 1 + else: + nxf = int(nxf) + + slopes_tile_wise = [] + for i in range(nxf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + elif i > 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + + if index_x_u > img_path.shape[1]: + index_x_u = img_path.shape[1] + index_x_d = img_path.shape[1] - length_x + + # img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + + sigma = 2 + try: + slope_xline = return_deskew_slop(img_xline, sigma, dir_of_all=dir_of_all, f_name=f_name) + except: + slope_xline = 0 + + if abs(slope_region) < 25 and abs(slope_xline) > 25: + slope_xline = [slope_region][0] + # if abs(slope_region)>70 and abs(slope_xline)<25: + # slope_xline=[slope_region][0] + slopes_tile_wise.append(slope_xline) + # print(slope_xline,'xlineeee') + img_line_rotated = rotate_image(img_xline, slope_xline) + img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 + + # print(slopes_tile_wise,'slopes_tile_wise') + img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] + + img_patch_ineterst_revised = np.zeros(img_patch_ineterst.shape) + + for i in range(nxf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + elif i > 0: + index_x_d = i * width_mid + index_x_u = index_x_d + length_x + + if index_x_u > img_path.shape[1]: + index_x_u = img_path.shape[1] + index_x_d = img_path.shape[1] - length_x + + img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + + img_int = np.zeros((img_xline.shape[0], img_xline.shape[1])) + img_int[:, :] = img_xline[:, :] # img_patch_org[:,:,0] + + img_resized = np.zeros((int(img_int.shape[0] * (1.2)), int(img_int.shape[1] * (3)))) + + img_resized[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], int(img_int.shape[1] * (1)) : int(img_int.shape[1] * (1)) + img_int.shape[1]] = img_int[:, :] + # plt.imshow(img_xline) + # plt.show() + img_line_rotated = rotate_image(img_resized, slopes_tile_wise[i]) + img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 + + img_patch_seperated = seperate_lines_new_inside_teils2(img_line_rotated, 0) + + img_patch_seperated_returned = rotate_image(img_patch_seperated, -slopes_tile_wise[i]) + img_patch_seperated_returned[:, :][img_patch_seperated_returned[:, :] != 0] = 1 + + img_patch_seperated_returned_true_size = img_patch_seperated_returned[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], int(img_int.shape[1] * (1)) : int(img_int.shape[1] * (1)) + img_int.shape[1]] + + img_patch_seperated_returned_true_size = img_patch_seperated_returned_true_size[:, margin : length_x - margin] + img_patch_ineterst_revised[:, index_x_d + margin : index_x_u - margin] = img_patch_seperated_returned_true_size + + # plt.imshow(img_patch_ineterst_revised) + # plt.show() + return img_patch_ineterst_revised + +def return_deskew_slop(img_patch_org, sigma_des, main_page=False, dir_of_all=None, f_name=None): + + + if main_page and dir_of_all is not None: + + + plt.figure(figsize=(70,40)) + plt.rcParams['font.size']='50' + plt.subplot(1,2,1) + plt.imshow(img_patch_org) + plt.subplot(1,2,2) + plt.plot(gaussian_filter1d(img_patch_org.sum(axis=1), 3),np.array(range(len(gaussian_filter1d(img_patch_org.sum(axis=1), 3)))),linewidth=8) + plt.xlabel('Density of textline prediction in direction of X axis',fontsize=60) + plt.ylabel('Height',fontsize=60) + plt.yticks([0,len(gaussian_filter1d(img_patch_org.sum(axis=1), 3))]) + plt.gca().invert_yaxis() + + plt.savefig(os.path.join(dir_of_all, f_name+'_density_of_textline.png')) + #print(np.max(img_patch_org.sum(axis=0)) ,np.max(img_patch_org.sum(axis=1)),'axislar') + + #img_patch_org=resize_image(img_patch_org,int(img_patch_org.shape[0]*2.5),int(img_patch_org.shape[1]/2.5)) + + #print(np.max(img_patch_org.sum(axis=0)) ,np.max(img_patch_org.sum(axis=1)),'axislar2') + + img_int=np.zeros((img_patch_org.shape[0],img_patch_org.shape[1])) + img_int[:,:]=img_patch_org[:,:]#img_patch_org[:,:,0] + + + + max_shape=np.max(img_int.shape) + img_resized=np.zeros((int( max_shape*(1.1) ) , int( max_shape*(1.1) ) )) + + + onset_x=int((img_resized.shape[1]-img_int.shape[1])/2.) + onset_y=int((img_resized.shape[0]-img_int.shape[0])/2.) + + + #img_resized=np.zeros((int( img_int.shape[0]*(1.8) ) , int( img_int.shape[1]*(2.6) ) )) + + + + #img_resized[ int( img_int.shape[0]*(.4)):int( img_int.shape[0]*(.4))+img_int.shape[0] , int( img_int.shape[1]*(.8)):int( img_int.shape[1]*(.8))+img_int.shape[1] ]=img_int[:,:] + img_resized[ onset_y:onset_y+img_int.shape[0] , onset_x:onset_x+img_int.shape[1] ]=img_int[:,:] + + #print(img_resized.shape,'img_resizedshape') + #plt.imshow(img_resized) + #plt.show() + + if main_page and img_patch_org.shape[1]>img_patch_org.shape[0]: + + #plt.imshow(img_resized) + #plt.show() + angels=np.array([-45, 0 , 45 , 90 , ])#np.linspace(-12,12,100)#np.array([0 , 45 , 90 , -45]) + + #res=[] + #num_of_peaks=[] + #index_cor=[] + var_res=[] + + #indexer=0 + for rot in angels: + img_rot=rotate_image(img_resized,rot) + #plt.imshow(img_rot) + #plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + + + #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(var_spectrum,'var_spectrum') + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + ##print(rot,var_spectrum,'var_spectrum') + #res_me=np.mean(neg_peaks) + #if res_me==0: + #res_me=1000000000000000000000 + #else: + #pass + + #res_num=len(neg_peaks) + except: + #res_me=1000000000000000000000 + #res_num=0 + var_spectrum=0 + #if self.isNaN(res_me): + #pass + #else: + #res.append( res_me ) + #var_res.append(var_spectrum) + #num_of_peaks.append( res_num ) + #index_cor.append(indexer) + #indexer=indexer+1 + + var_res.append(var_spectrum) + #index_cor.append(indexer) + #indexer=indexer+1 + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + + angels=np.linspace(ang_int-22.5,ang_int+22.5,100) + + #res=[] + #num_of_peaks=[] + #index_cor=[] + var_res=[] + + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + elif main_page and img_patch_org.shape[1]<=img_patch_org.shape[0]: + + #plt.imshow(img_resized) + #plt.show() + angels=np.linspace(-12,12,100)#np.array([0 , 45 , 90 , -45]) + + + var_res=[] + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + #plt.imshow(img_rot) + #plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + + + #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(var_spectrum,'var_spectrum') + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + if dir_of_all is not None: + #print('galdi?') + plt.figure(figsize=(60,30)) + plt.rcParams['font.size']='50' + plt.plot(angels,np.array(var_res),'-o',markersize=25,linewidth=4) + plt.xlabel('angle',fontsize=50) + plt.ylabel('variance of sum of rotated textline in direction of x axis',fontsize=50) + + plt.plot(angels[np.argmax(var_res)],var_res[np.argmax(np.array(var_res))] ,'*',markersize=50,label='Angle of deskewing=' +str("{:.2f}".format(angels[np.argmax(var_res)]))+r'$\degree$') + plt.legend(loc='best') + plt.savefig(os.path.join(dir_of_all,f_name+'_rotation_angle.png')) + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + + early_slope_edge=11 + if abs(ang_int)>early_slope_edge and ang_int<0: + + angels=np.linspace(-90,-12,100) + + var_res=[] + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + elif abs(ang_int)>early_slope_edge and ang_int>0: + + angels=np.linspace(90,12,100) + + + var_res=[] + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(indexer,'indexer') + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + else: + + + angels=np.linspace(-25,25,60) + + var_res=[] + + indexer=0 + for rot in angels: + img_rot=rotate_image(img_resized,rot) + #plt.imshow(img_rot) + #plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + + + #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(var_spectrum,'var_spectrum') + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + #plt.plot(var_res) + #plt.show() + + ##plt.plot(mom3_res) + ##plt.show() + #print(ang_int,'ang_int111') + + early_slope_edge=22 + if abs(ang_int)>early_slope_edge and ang_int<0: + + angels=np.linspace(-90,-25,60) + + var_res=[] + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + elif abs(ang_int)>early_slope_edge and ang_int>0: + + angels=np.linspace(90,25,60) + + var_res=[] + + indexer=0 + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + #res_me=np.mean(self.find_num_col_deskew(img_rot,sigma_des,2.0 )) + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(indexer,'indexer') + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + + try: + var_res=np.array(var_res) + + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + return ang_int + diff --git a/setup.py b/setup.py index 5901c19..a2cb66c 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( install_requires=install_requires, entry_points={ 'console_scripts': [ - 'eynollah=sbb_newspapers_org_image.eynollah:main', + 'eynollah=sbb_newspapers_org_image.cli:main', # 'ocrd-eynollah=eynollah.ocrd_cli:cli', ] }, diff --git a/tests/resources/kant_aufklaerung_1784_0020.tif b/tests/resources/kant_aufklaerung_1784_0020.tif new file mode 100644 index 0000000..13db797 Binary files /dev/null and b/tests/resources/kant_aufklaerung_1784_0020.tif differ diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..7d1b381 --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,7 @@ +def test_utils_import(): + import sbb_newspapers_org_image.utils + import sbb_newspapers_org_image.utils.contour + import sbb_newspapers_org_image.utils.drop_capitals + import sbb_newspapers_org_image.utils.drop_capitals + import sbb_newspapers_org_image.utils.is_nan + import sbb_newspapers_org_image.utils.rotate