mirror of
https://github.com/qurator-spk/eynollah.git
synced 2025-08-29 11:59:55 +02:00
mb reading order now can be done faster. Text regions are clustered using dilation, and mb reading order needs to be implemented for fewer regions
This commit is contained in:
vahidrezanezhad
parent
7a34bbb493
commit
848156dd9d
2 changed files with 163 additions and 20 deletions
|
|
@ -32,6 +32,7 @@ from numba import cuda
|
|||
from skimage.morphology import skeletonize
|
||||
from ocrd import OcrdPage
|
||||
from ocrd_utils import getLogger, tf_disable_interactive_logs
|
||||
import statistics
|
||||
|
||||
try:
|
||||
import torch
|
||||
|
|
@ -797,7 +798,7 @@ class Eynollah:
|
|||
self, patches, img, model,
|
||||
n_batch_inference=1, marginal_of_patch_percent=0.1,
|
||||
thresholding_for_some_classes_in_light_version=False,
|
||||
thresholding_for_artificial_class_in_light_version=False, threshold_art_class_textline=0.1):
|
||||
thresholding_for_artificial_class_in_light_version=False, thresholding_for_fl_light_version=False, threshold_art_class_textline=0.1):
|
||||
|
||||
self.logger.debug("enter do_prediction")
|
||||
img_height_model = model.layers[-1].output_shape[1]
|
||||
|
|
@ -822,6 +823,15 @@ class Eynollah:
|
|||
skeleton_art = skeleton_art*1
|
||||
|
||||
seg[skeleton_art==1]=2
|
||||
|
||||
if thresholding_for_fl_light_version:
|
||||
seg_header = label_p_pred[0,:,:,2]
|
||||
|
||||
seg_header[seg_header<0.2] = 0
|
||||
seg_header[seg_header>0] =1
|
||||
|
||||
seg[seg_header==1]=2
|
||||
|
||||
seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2)
|
||||
prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8)
|
||||
return prediction_true
|
||||
|
|
@ -1613,10 +1623,11 @@ class Eynollah:
|
|||
model_region = self.model_region_fl if patches else self.model_region_fl_np
|
||||
|
||||
if self.light_version:
|
||||
pass
|
||||
thresholding_for_fl_light_version = True
|
||||
elif not patches:
|
||||
img = otsu_copy_binary(img).astype(np.uint8)
|
||||
prediction_regions = None
|
||||
thresholding_for_fl_light_version = False
|
||||
elif cols:
|
||||
img = otsu_copy_binary(img).astype(np.uint8)
|
||||
if cols == 1:
|
||||
|
|
@ -1632,7 +1643,7 @@ class Eynollah:
|
|||
else:
|
||||
img = resize_image(img, int(img_height_h * 2500 / float(img_width_h)), 2500).astype(np.uint8)
|
||||
|
||||
prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1, n_batch_inference=3)
|
||||
prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1, n_batch_inference=3, thresholding_for_fl_light_version=thresholding_for_fl_light_version)
|
||||
prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h)
|
||||
self.logger.debug("exit extract_text_regions")
|
||||
return prediction_regions, prediction_regions
|
||||
|
|
@ -3544,25 +3555,94 @@ class Eynollah:
|
|||
return model
|
||||
|
||||
def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p):
|
||||
#cv2.imwrite('textregions.png', text_regions_p*50)
|
||||
min_cont_size_to_be_dilated = 10
|
||||
if len(contours_only_text_parent)>min_cont_size_to_be_dilated:
|
||||
ver_kernel = np.ones((5, 1), dtype=np.uint8)
|
||||
|
||||
cx_conts, cy_conts, x_min_conts, x_max_conts, y_min_conts, y_max_conts, _ = find_new_features_of_contours(contours_only_text_parent)
|
||||
args_cont_located = np.array(range(len(contours_only_text_parent)))
|
||||
|
||||
diff_y_conts = np.abs(y_max_conts[:]-y_min_conts)
|
||||
diff_x_conts = np.abs(x_max_conts[:]-x_min_conts)
|
||||
|
||||
mean_x = statistics.mean(diff_x_conts)
|
||||
median_x = statistics.median(diff_x_conts)
|
||||
|
||||
|
||||
diff_x_ratio= diff_x_conts/mean_x
|
||||
|
||||
args_cont_located_excluded = args_cont_located[diff_x_ratio>=1.3]
|
||||
args_cont_located_included = args_cont_located[diff_x_ratio<1.3]
|
||||
|
||||
contours_only_text_parent_excluded = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]>=1.3]#contours_only_text_parent[diff_x_ratio>=1.3]
|
||||
contours_only_text_parent_included = [contours_only_text_parent[ind] for ind in range(len(contours_only_text_parent)) if diff_x_ratio[ind]<1.3]#contours_only_text_parent[diff_x_ratio<1.3]
|
||||
|
||||
|
||||
cx_conts_excluded = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]>=1.3]#cx_conts[diff_x_ratio>=1.3]
|
||||
cx_conts_included = [cx_conts[ind] for ind in range(len(cx_conts)) if diff_x_ratio[ind]<1.3]#cx_conts[diff_x_ratio<1.3]
|
||||
|
||||
cy_conts_excluded = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]>=1.3]#cy_conts[diff_x_ratio>=1.3]
|
||||
cy_conts_included = [cy_conts[ind] for ind in range(len(cy_conts)) if diff_x_ratio[ind]<1.3]#cy_conts[diff_x_ratio<1.3]
|
||||
|
||||
#print(diff_x_ratio, 'ratio')
|
||||
text_regions_p = text_regions_p.astype('uint8')
|
||||
|
||||
if len(contours_only_text_parent_excluded)>0:
|
||||
textregion_par = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1])).astype('uint8')
|
||||
textregion_par = cv2.fillPoly(textregion_par, pts=contours_only_text_parent_included, color=(1,1))
|
||||
else:
|
||||
textregion_par = (text_regions_p[:,:]==1)*1
|
||||
textregion_par = textregion_par.astype('uint8')
|
||||
|
||||
|
||||
text_regions_p_textregions_dilated = cv2.dilate(textregion_par , ver_kernel, iterations=8)
|
||||
text_regions_p_textregions_dilated[text_regions_p[:,:]>1] = 0
|
||||
|
||||
#cv2.imwrite('textregions_dilated.png', text_regions_p_textregions_dilated*255)
|
||||
|
||||
|
||||
contours_only_dilated, hir_on_text_dilated = return_contours_of_image(text_regions_p_textregions_dilated)
|
||||
contours_only_dilated = return_parent_contours(contours_only_dilated, hir_on_text_dilated)
|
||||
|
||||
indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located = self.return_indexes_of_contours_loctaed_inside_another_list_of_contours(contours_only_dilated, contours_only_text_parent_included, cx_conts_included, cy_conts_included, args_cont_located_included)
|
||||
|
||||
|
||||
if len(args_cont_located_excluded)>0:
|
||||
for ind in args_cont_located_excluded:
|
||||
indexes_of_located_cont.append(np.array([ind]))
|
||||
contours_only_dilated.append(contours_only_text_parent[ind])
|
||||
center_y_coordinates_of_located.append(0)
|
||||
|
||||
array_list = [np.array([elem]) if isinstance(elem, int) else elem for elem in indexes_of_located_cont]
|
||||
flattened_array = np.concatenate([arr.ravel() for arr in array_list])
|
||||
#print(len( np.unique(flattened_array)), 'indexes_of_located_cont uniques')
|
||||
|
||||
missing_textregions = list( set(np.array(range(len(contours_only_text_parent))) ) - set(np.unique(flattened_array)) )
|
||||
#print(missing_textregions, 'missing_textregions')
|
||||
|
||||
for ind in missing_textregions:
|
||||
indexes_of_located_cont.append(np.array([ind]))
|
||||
contours_only_dilated.append(contours_only_text_parent[ind])
|
||||
center_y_coordinates_of_located.append(0)
|
||||
|
||||
|
||||
if contours_only_text_parent_h:
|
||||
for vi in range(len(contours_only_text_parent_h)):
|
||||
indexes_of_located_cont.append(int(vi+len(contours_only_text_parent)))
|
||||
|
||||
array_list = [np.array([elem]) if isinstance(elem, int) else elem for elem in indexes_of_located_cont]
|
||||
flattened_array = np.concatenate([arr.ravel() for arr in array_list])
|
||||
|
||||
y_len = text_regions_p.shape[0]
|
||||
x_len = text_regions_p.shape[1]
|
||||
|
||||
|
||||
img_poly = np.zeros((y_len,x_len), dtype='uint8')
|
||||
img_poly[text_regions_p[:,:]==1] = 1
|
||||
img_poly[text_regions_p[:,:]==2] = 2
|
||||
img_poly[text_regions_p[:,:]==3] = 4
|
||||
img_poly[text_regions_p[:,:]==6] = 5
|
||||
|
||||
|
||||
###temp
|
||||
##sep_mask = (img_poly==5)*1
|
||||
##sep_mask = sep_mask.astype('uint8')
|
||||
##sep_mask = cv2.erode(sep_mask, kernel=KERNEL, iterations=2)
|
||||
##img_poly[img_poly==5] = 0
|
||||
##img_poly[sep_mask==1] = 5
|
||||
###
|
||||
|
||||
img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8')
|
||||
if contours_only_text_parent_h:
|
||||
_, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours(
|
||||
|
|
@ -3570,7 +3650,15 @@ class Eynollah:
|
|||
for j in range(len(cy_main)):
|
||||
img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12,
|
||||
int(x_min_main[j]):int(x_max_main[j])] = 1
|
||||
co_text_all_org = contours_only_text_parent + contours_only_text_parent_h
|
||||
if len(contours_only_text_parent)>min_cont_size_to_be_dilated:
|
||||
co_text_all = contours_only_dilated + contours_only_text_parent_h
|
||||
else:
|
||||
co_text_all = contours_only_text_parent + contours_only_text_parent_h
|
||||
else:
|
||||
co_text_all_org = contours_only_text_parent
|
||||
if len(contours_only_text_parent)>min_cont_size_to_be_dilated:
|
||||
co_text_all = contours_only_dilated
|
||||
else:
|
||||
co_text_all = contours_only_text_parent
|
||||
|
||||
|
|
@ -3651,9 +3739,27 @@ class Eynollah:
|
|||
break
|
||||
|
||||
ordered = [i[0] for i in ordered]
|
||||
region_ids = ['region_%04d' % i for i in range(len(co_text_all))]
|
||||
|
||||
if len(contours_only_text_parent)>min_cont_size_to_be_dilated:
|
||||
org_contours_indexes = []
|
||||
for ind in range(len(ordered)):
|
||||
region_with_curr_order = ordered[ind]
|
||||
if region_with_curr_order < len(contours_only_dilated):
|
||||
if np.isscalar(indexes_of_located_cont[region_with_curr_order]):
|
||||
org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]]
|
||||
else:
|
||||
arg_sort_located_cont = np.argsort(center_y_coordinates_of_located[region_with_curr_order])
|
||||
org_contours_indexes = org_contours_indexes + list(np.array(indexes_of_located_cont[region_with_curr_order])[arg_sort_located_cont]) ##org_contours_indexes + list (
|
||||
else:
|
||||
org_contours_indexes = org_contours_indexes + [indexes_of_located_cont[region_with_curr_order]]
|
||||
|
||||
region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))]
|
||||
return org_contours_indexes, region_ids
|
||||
else:
|
||||
region_ids = ['region_%04d' % i for i in range(len(co_text_all_org))]
|
||||
return ordered, region_ids
|
||||
|
||||
|
||||
def return_start_and_end_of_common_text_of_textline_ocr(self, textline_image, ind_tot):
|
||||
width = np.shape(textline_image)[1]
|
||||
height = np.shape(textline_image)[0]
|
||||
|
|
@ -4294,6 +4400,29 @@ class Eynollah:
|
|||
|
||||
return contours
|
||||
|
||||
def return_indexes_of_contours_loctaed_inside_another_list_of_contours(self, contours, contours_loc, cx_main_loc, cy_main_loc, indexes_loc):
|
||||
indexes_of_located_cont = []
|
||||
center_x_coordinates_of_located = []
|
||||
center_y_coordinates_of_located = []
|
||||
#M_main_tot = [cv2.moments(contours_loc[j])
|
||||
#for j in range(len(contours_loc))]
|
||||
#cx_main_loc = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))]
|
||||
#cy_main_loc = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))]
|
||||
|
||||
for ij in range(len(contours)):
|
||||
results = [cv2.pointPolygonTest(contours[ij], (cx_main_loc[ind], cy_main_loc[ind]), False)
|
||||
for ind in range(len(cy_main_loc)) ]
|
||||
results = np.array(results)
|
||||
indexes_in = np.where((results == 0) | (results == 1))
|
||||
indexes = indexes_loc[indexes_in]# [(results == 0) | (results == 1)]#np.where((results == 0) | (results == 1))
|
||||
|
||||
indexes_of_located_cont.append(indexes)
|
||||
center_x_coordinates_of_located.append(np.array(cx_main_loc)[indexes_in] )
|
||||
center_y_coordinates_of_located.append(np.array(cy_main_loc)[indexes_in] )
|
||||
|
||||
return indexes_of_located_cont, center_x_coordinates_of_located, center_y_coordinates_of_located
|
||||
|
||||
|
||||
def filter_contours_without_textline_inside(
|
||||
self, contours,text_con_org, contours_textline, contours_only_text_parent_d_ordered, conf_contours_textregions):
|
||||
###contours_txtline_of_all_textregions = []
|
||||
|
|
@ -4986,8 +5115,10 @@ class Eynollah:
|
|||
|
||||
if self.full_layout:
|
||||
if self.reading_order_machine_based:
|
||||
tror = time.time()
|
||||
order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model(
|
||||
contours_only_text_parent, contours_only_text_parent_h, text_regions_p)
|
||||
print('time spend for mb ro', time.time()-tror)
|
||||
else:
|
||||
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||
order_text_new, id_of_texts_tot = self.do_order_of_regions(
|
||||
|
|
@ -5619,8 +5750,15 @@ class Eynollah_ocr:
|
|||
mask_poly = np.zeros(img.shape)
|
||||
mask_poly = cv2.fillPoly(mask_poly, pts=[textline_coords], color=(1, 1, 1))
|
||||
|
||||
|
||||
mask_poly = mask_poly[y:y+h, x:x+w, :]
|
||||
img_crop = img_poly_on_img[y:y+h, x:x+w, :]
|
||||
|
||||
if angle_degrees<=15:
|
||||
if mask_poly[:,:,0].sum() /float(w*h) < 0.6 and w_scaled > 520:
|
||||
cv2.imwrite(file_name+'_desk.png', img_crop)
|
||||
|
||||
print(file_name, angle_degrees,w*h , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w*h) , 'didi')
|
||||
if not self.do_not_mask_with_textline_contour:
|
||||
if angle_degrees > 15:
|
||||
better_des_slope = self.get_orientation_moments(textline_coords)
|
||||
|
|
@ -5635,13 +5773,18 @@ class Eynollah_ocr:
|
|||
mask_poly = mask_poly[y_n:y_n+h_n, x_n:x_n+w_n, :]
|
||||
img_crop = img_crop[y_n:y_n+h_n, x_n:x_n+w_n, :]
|
||||
|
||||
if mask_poly[:,:,0].sum() /float(w_n*h_n) < 0.6 and w_scaled > 520:
|
||||
cv2.imwrite(file_name+'_desk.png', img_crop)
|
||||
|
||||
print(file_name,w_n*h_n , mask_poly[:,:,0].sum(), mask_poly[:,:,0].sum() /float(w_n*h_n) , 'ikiiiiii')
|
||||
|
||||
img_crop[mask_poly==0] = 255
|
||||
|
||||
if self.prediction_with_both_of_rgb_and_bin:
|
||||
img_crop_bin[mask_poly==0] = 255
|
||||
|
||||
if not self.export_textline_images_and_text:
|
||||
if w_scaled < 1.5*image_width:
|
||||
if w_scaled < 640:#1.5*image_width:
|
||||
img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width)
|
||||
cropped_lines.append(img_fin)
|
||||
if angle_degrees > 15:
|
||||
|
|
|
|||
|
|
@ -992,7 +992,7 @@ def check_any_text_region_in_model_one_is_main_or_header_light(
|
|||
(regions_model_full[:,:,0]==2)).sum()
|
||||
pixels_main = all_pixels - pixels_header
|
||||
|
||||
if (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ):
|
||||
if ( (pixels_header/float(pixels_main)>=0.6) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ) and ( (length_con[ii]/float(height_con[ii]) )<=3 )) or ( (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=3 ) ):
|
||||
regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2
|
||||
contours_only_text_parent_head.append(con)
|
||||
if contours_only_text_parent_d_ordered is not None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue