@ -38,7 +38,8 @@ sys.stderr = stderr
tf . get_logger ( ) . setLevel ( " ERROR " )
warnings . filterwarnings ( " ignore " )
import matplotlib . pyplot as plt
from tensorflow . python . keras . backend import set_session
# use tf1 compatibility for keras backend
from tensorflow . compat . v1 . keras . backend import set_session
from tensorflow . keras import layers
from . utils . contour import (
@ -158,6 +159,7 @@ class Eynollah:
dir_out = None ,
dir_in = None ,
dir_of_cropped_images = None ,
extract_only_images = False ,
dir_of_layout = None ,
dir_of_deskewed = None ,
dir_of_all = None ,
@ -211,6 +213,8 @@ class Eynollah:
self . input_binary = input_binary
self . allow_scaling = allow_scaling
self . headers_off = headers_off
self . light_version = light_version
self . extract_only_images = extract_only_images
self . ignore_page_extraction = ignore_page_extraction
self . skip_layout_and_reading_order = skip_layout_and_reading_order
self . ocr = do_ocr
@ -250,6 +254,7 @@ class Eynollah:
self . model_page_dir = dir_models + " /eynollah-page-extraction_20210425 "
self . model_region_dir_p_ens = dir_models + " /eynollah-main-regions-ensembled_20210425 "
self . model_region_dir_p_ens_light = dir_models + " /eynollah-main-regions_20220314 "
self . model_region_dir_p_ens_light_only_images_extraction = dir_models + " /eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18 "
self . model_reading_order_machine_dir = dir_models + " /model_ens_reading_order_machine_based "
self . model_region_dir_p_1_2_sp_np = dir_models + " /modelens_e_l_all_sp_0_1_2_3_4_171024 " #"/modelens_12sp_elay_0_3_4__3_6_n"#"/modelens_earlylayout_12spaltige_2_3_5_6_7_8"#"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18"#"/modelens_1_2_4_5_early_lay_1_2_spaltige"#"/model_3_eraly_layout_no_patches_1_2_spaltige"
##self.model_region_dir_fully_new = dir_models + "/model_2_full_layout_new_trans"
@ -288,7 +293,23 @@ class Eynollah:
self . ls_imgs = os . listdir ( self . dir_in )
if dir_in and not light_version :
if dir_in and self . extract_only_images :
config = tf . compat . v1 . ConfigProto ( )
config . gpu_options . allow_growth = True
session = tf . compat . v1 . Session ( config = config )
set_session ( session )
self . model_page = self . our_load_model ( self . model_page_dir )
self . model_classifier = self . our_load_model ( self . model_dir_of_col_classifier )
self . model_bin = self . our_load_model ( self . model_dir_of_binarization )
#self.model_textline = self.our_load_model(self.model_textline_dir)
self . model_region = self . our_load_model ( self . model_region_dir_p_ens_light_only_images_extraction )
#self.model_region_fl_np = self.our_load_model(self.model_region_dir_fully_np)
#self.model_region_fl = self.our_load_model(self.model_region_dir_fully)
self . ls_imgs = os . listdir ( self . dir_in )
if dir_in and not ( light_version or self . extract_only_images ) :
config = tf . compat . v1 . ConfigProto ( )
config . gpu_options . allow_growth = True
session = tf . compat . v1 . Session ( config = config )
@ -534,6 +555,27 @@ class Eynollah:
return img_new , num_column_is_classified
def calculate_width_height_by_columns_extract_only_images ( self , img , num_col , width_early , label_p_pred ) :
self . logger . debug ( " enter calculate_width_height_by_columns " )
if num_col == 1 :
img_w_new = 700
elif num_col == 2 :
img_w_new = 900
elif num_col == 3 :
img_w_new = 1500
elif num_col == 4 :
img_w_new = 1800
elif num_col == 5 :
img_w_new = 2200
elif num_col == 6 :
img_w_new = 2500
img_h_new = int ( img . shape [ 0 ] / float ( img . shape [ 1 ] ) * img_w_new )
img_new = resize_image ( img , img_h_new , img_w_new )
num_column_is_classified = True
return img_new , num_column_is_classified
def resize_image_with_column_classifier ( self , is_image_enhanced , img_bin ) :
self . logger . debug ( " enter resize_image_with_column_classifier " )
if self . input_binary :
@ -690,6 +732,7 @@ class Eynollah:
self . logger . info ( " Found %d columns ( %s ) " , num_col , np . around ( label_p_pred , decimals = 5 ) )
if not self . extract_only_images :
if dpi < DPI_THRESHOLD :
if light_version and num_col in ( 1 , 2 ) :
img_new , num_column_is_classified = self . calculate_width_height_by_columns_1_2 ( img , num_col , width_early , label_p_pred )
@ -709,6 +752,10 @@ class Eynollah:
num_column_is_classified = True
image_res = np . copy ( img )
is_image_enhanced = False
else :
num_column_is_classified = True
image_res = np . copy ( img )
is_image_enhanced = False
self . logger . debug ( " exit resize_and_enhance_image_with_column_classifier " )
return is_image_enhanced , img , image_res , num_col , num_column_is_classified , img_bin
@ -1191,132 +1238,21 @@ class Eynollah:
batch_indexer = batch_indexer + 1
#img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :]
#label_p_pred = model.predict(img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2]),
#verbose=0)
#seg = np.argmax(label_p_pred, axis=3)[0]
######seg_not_base = label_p_pred[0,:,:,4]
########seg2 = -label_p_pred[0,:,:,2]
######seg_not_base[seg_not_base>0.03] =1
######seg_not_base[seg_not_base<1] =0
######seg_test = label_p_pred[0,:,:,1]
########seg2 = -label_p_pred[0,:,:,2]
######seg_test[seg_test>0.75] =1
######seg_test[seg_test<1] =0
######seg_line = label_p_pred[0,:,:,3]
########seg2 = -label_p_pred[0,:,:,2]
######seg_line[seg_line>0.1] =1
######seg_line[seg_line<1] =0
######seg_background = label_p_pred[0,:,:,0]
########seg2 = -label_p_pred[0,:,:,2]
######seg_background[seg_background>0.25] =1
######seg_background[seg_background<1] =0
##seg = seg+seg2
#seg = label_p_pred[0,:,:,2]
#seg[seg>0.4] =1
#seg[seg<1] =0
##plt.imshow(seg_test)
##plt.show()
##plt.imshow(seg_background)
##plt.show()
#seg[seg==1]=0
#seg[seg_test==1]=1
######seg[seg_not_base==1]=4
######seg[seg_background==1]=0
######seg[(seg_line==1) & (seg==0)]=3
#seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2)
#if i == 0 and j == 0:
#seg_color = seg_color[0 : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :]
#seg = seg[0 : seg.shape[0] - margin, 0 : seg.shape[1] - margin]
#mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin] = seg
#prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg_color
#elif i == nxf - 1 and j == nyf - 1:
#seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - 0, :]
#seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - 0]
#mask_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0] = seg
#prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0, :] = seg_color
#elif i == 0 and j == nyf - 1:
#seg_color = seg_color[margin : seg_color.shape[0] - 0, 0 : seg_color.shape[1] - margin, :]
#seg = seg[margin : seg.shape[0] - 0, 0 : seg.shape[1] - margin]
#mask_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin] = seg
#prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin, :] = seg_color
#elif i == nxf - 1 and j == 0:
#seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :]
#seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - 0]
#mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0] = seg
#prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg_color
#elif i == 0 and j != 0 and j != nyf - 1:
#seg_color = seg_color[margin : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :]
#seg = seg[margin : seg.shape[0] - margin, 0 : seg.shape[1] - margin]
#mask_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin] = seg
#prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg_color
#elif i == nxf - 1 and j != 0 and j != nyf - 1:
#seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :]
#seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - 0]
#mask_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0] = seg
#prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg_color
#elif i != 0 and i != nxf - 1 and j == 0:
#seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :]
#seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - margin]
#mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin] = seg
#prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg_color
#elif i != 0 and i != nxf - 1 and j == nyf - 1:
#seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - margin, :]
#seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - margin]
#mask_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin] = seg
#prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin, :] = seg_color
#else:
#seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :]
#seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - margin]
#mask_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin] = seg
#prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg_color
if batch_indexer == n_batch_inference :
label_p_pred = model . predict ( img_patch , verbose = 0 )
seg = np . argmax ( label_p_pred , axis = 3 )
if thresholding_for_some_classes_in_light_version :
seg_art = label_p_pred [ : , : , : , 4 ]
seg_art [ seg_art < 0.2 ] = 0
seg_art [ seg_art > 0 ] = 1
###seg[seg_art==1]=4
##seg_not_base = label_p_pred[:,:,:,4]
##seg_not_base[seg_not_base>0.03] =1
##seg_not_base[seg_not_base<1] =0
seg_line = label_p_pred [ : , : , : , 3 ]
seg_line [ seg_line > 0.1 ] = 1
seg_line [ seg_line < 1 ] = 0
##seg_background = label_p_pred[:,:,:,0]
##seg_background[seg_background>0.25] =1
##seg_background[seg_background<1] =0
seg [ seg_art == 1 ] = 4
##seg[seg_background==1]=0
seg [ ( seg_line == 1 ) & ( seg == 0 ) ] = 3
if thresholding_for_artificial_class_in_light_version :
seg_art = label_p_pred [ : , : , : , 2 ]
@ -1384,20 +1320,15 @@ class Eynollah:
seg = np . argmax ( label_p_pred , axis = 3 )
if thresholding_for_some_classes_in_light_version :
seg_ not_base = label_p_pred [ : , : , : , 4 ]
seg_ not_base[ seg_not_base > 0.03 ] = 1
seg_ not_base[ seg_not_base < 1 ] = 0
seg_ art = label_p_pred [ : , : , : , 4 ]
seg_ art[ seg_art < 0.2 ] = 0
seg_ art[ seg_art > 0 ] = 1
seg_line = label_p_pred [ : , : , : , 3 ]
seg_line [ seg_line > 0.1 ] = 1
seg_line [ seg_line < 1 ] = 0
seg_background = label_p_pred [ : , : , : , 0 ]
seg_background [ seg_background > 0.25 ] = 1
seg_background [ seg_background < 1 ] = 0
seg [ seg_not_base == 1 ] = 4
seg [ seg_background == 1 ] = 0
seg [ seg_art == 1 ] = 4
seg [ ( seg_line == 1 ) & ( seg == 0 ) ] = 3
if thresholding_for_artificial_class_in_light_version :
@ -1449,7 +1380,6 @@ class Eynollah:
indexer_inside_batch = indexer_inside_batch + 1
list_i_s = [ ]
list_j_s = [ ]
list_x_u = [ ]
@ -1458,7 +1388,6 @@ class Eynollah:
list_y_d = [ ]
batch_indexer = 0
img_patch = np . zeros ( ( n_batch_inference , img_height_model , img_width_model , 3 ) )
prediction_true = prediction_true . astype ( np . uint8 )
@ -2224,6 +2153,125 @@ class Eynollah:
q . put ( slopes_sub )
poly . put ( poly_sub )
box_sub . put ( boxes_sub_new )
def get_regions_light_v_extract_only_images ( self , img , is_image_enhanced , num_col_classifier ) :
self . logger . debug ( " enter get_regions_extract_images_only " )
erosion_hurts = False
img_org = np . copy ( img )
img_height_h = img_org . shape [ 0 ]
img_width_h = img_org . shape [ 1 ]
if num_col_classifier == 1 :
img_w_new = 700
elif num_col_classifier == 2 :
img_w_new = 900
elif num_col_classifier == 3 :
img_w_new = 1500
elif num_col_classifier == 4 :
img_w_new = 1800
elif num_col_classifier == 5 :
img_w_new = 2200
elif num_col_classifier == 6 :
img_w_new = 2500
img_h_new = int ( img . shape [ 0 ] / float ( img . shape [ 1 ] ) * img_w_new )
img_resized = resize_image ( img , img_h_new , img_w_new )
if not self . dir_in :
model_region , session_region = self . start_new_session_and_model ( self . model_region_dir_p_ens_light_only_images_extraction )
prediction_regions_org = self . do_prediction_new_concept ( True , img_resized , model_region )
else :
prediction_regions_org = self . do_prediction_new_concept ( True , img_resized , self . model_region )
#plt.imshow(prediction_regions_org[:,:,0])
#plt.show()
prediction_regions_org = resize_image ( prediction_regions_org , img_height_h , img_width_h )
image_page , page_coord , cont_page = self . extract_page ( )
prediction_regions_org = prediction_regions_org [ page_coord [ 0 ] : page_coord [ 1 ] , page_coord [ 2 ] : page_coord [ 3 ] ]
prediction_regions_org = prediction_regions_org [ : , : , 0 ]
mask_lines_only = ( prediction_regions_org [ : , : ] == 3 ) * 1
mask_texts_only = ( prediction_regions_org [ : , : ] == 1 ) * 1
mask_images_only = ( prediction_regions_org [ : , : ] == 2 ) * 1
polygons_lines_xml , hir_lines_xml = return_contours_of_image ( mask_lines_only )
polygons_lines_xml = textline_con_fil = filter_contours_area_of_image ( mask_lines_only , polygons_lines_xml , hir_lines_xml , max_area = 1 , min_area = 0.00001 )
polygons_of_only_texts = return_contours_of_interested_region ( mask_texts_only , 1 , 0.00001 )
polygons_of_only_lines = return_contours_of_interested_region ( mask_lines_only , 1 , 0.00001 )
text_regions_p_true = np . zeros ( prediction_regions_org . shape )
text_regions_p_true = cv2 . fillPoly ( text_regions_p_true , pts = polygons_of_only_lines , color = ( 3 , 3 , 3 ) )
text_regions_p_true [ : , : ] [ mask_images_only [ : , : ] == 1 ] = 2
text_regions_p_true = cv2 . fillPoly ( text_regions_p_true , pts = polygons_of_only_texts , color = ( 1 , 1 , 1 ) )
text_regions_p_true [ text_regions_p_true . shape [ 0 ] - 15 : text_regions_p_true . shape [ 0 ] , : ] = 0
text_regions_p_true [ : , text_regions_p_true . shape [ 1 ] - 15 : text_regions_p_true . shape [ 1 ] ] = 0
##polygons_of_images = return_contours_of_interested_region(text_regions_p_true, 2, 0.0001)
polygons_of_images = return_contours_of_interested_region ( text_regions_p_true , 2 , 0.001 )
image_boundary_of_doc = np . zeros ( ( text_regions_p_true . shape [ 0 ] , text_regions_p_true . shape [ 1 ] ) )
###image_boundary_of_doc[:6, :] = 1
###image_boundary_of_doc[text_regions_p_true.shape[0]-6:text_regions_p_true.shape[0], :] = 1
###image_boundary_of_doc[:, :6] = 1
###image_boundary_of_doc[:, text_regions_p_true.shape[1]-6:text_regions_p_true.shape[1]] = 1
#plt.imshow(image_boundary_of_doc)
#plt.show()
polygons_of_images_fin = [ ]
for ploy_img_ind in polygons_of_images :
"""
test_poly_image = np . zeros ( ( text_regions_p_true . shape [ 0 ] , text_regions_p_true . shape [ 1 ] ) )
test_poly_image = cv2 . fillPoly ( test_poly_image , pts = [ ploy_img_ind ] , color = ( 1 , 1 , 1 ) )
test_poly_image = test_poly_image [ : , : ] + image_boundary_of_doc [ : , : ]
test_poly_image_intersected_area = ( test_poly_image [ : , : ] == 2 ) * 1
test_poly_image_intersected_area = test_poly_image_intersected_area . sum ( )
if test_poly_image_intersected_area == 0 :
##polygons_of_images_fin.append(ploy_img_ind)
x , y , w , h = cv2 . boundingRect ( ploy_img_ind )
box = [ x , y , w , h ]
_ , page_coord_img = crop_image_inside_box ( box , text_regions_p_true )
#cont_page.append(np.array([[page_coord[2], page_coord[0]], [page_coord[3], page_coord[0]], [page_coord[3], page_coord[1]], [page_coord[2], page_coord[1]]]))
polygons_of_images_fin . append ( np . array ( [ [ page_coord_img [ 2 ] , page_coord_img [ 0 ] ] , [ page_coord_img [ 3 ] , page_coord_img [ 0 ] ] , [ page_coord_img [ 3 ] , page_coord_img [ 1 ] ] , [ page_coord_img [ 2 ] , page_coord_img [ 1 ] ] ] ) )
"""
x , y , w , h = cv2 . boundingRect ( ploy_img_ind )
if h < 150 or w < 150 :
pass
else :
box = [ x , y , w , h ]
_ , page_coord_img = crop_image_inside_box ( box , text_regions_p_true )
#cont_page.append(np.array([[page_coord[2], page_coord[0]], [page_coord[3], page_coord[0]], [page_coord[3], page_coord[1]], [page_coord[2], page_coord[1]]]))
polygons_of_images_fin . append ( np . array ( [ [ page_coord_img [ 2 ] , page_coord_img [ 0 ] ] , [ page_coord_img [ 3 ] , page_coord_img [ 0 ] ] , [ page_coord_img [ 3 ] , page_coord_img [ 1 ] ] , [ page_coord_img [ 2 ] , page_coord_img [ 1 ] ] ] ) )
return text_regions_p_true , erosion_hurts , polygons_lines_xml , polygons_of_images_fin , image_page , page_coord , cont_page
def get_regions_light_v ( self , img , is_image_enhanced , num_col_classifier , skip_layout_and_reading_order = False ) :
self . logger . debug ( " enter get_regions_light_v " )
t_in = time . time ( )
@ -3179,8 +3227,8 @@ class Eynollah:
prediction_table_erode = cv2 . erode ( prediction_table [ : , : , 0 ] , KERNEL , iterations = 20 )
prediction_table_erode = cv2 . dilate ( prediction_table_erode , KERNEL , iterations = 20 )
return prediction_table_erode . astype ( np . int16 )
def run_graphics_and_columns_light ( self , text_regions_p_1 , textline_mask_tot_ea , num_col_classifier , num_column_is_classified , erosion_hurts , img_bin_light ) :
def run_graphics_and_columns_light ( self , text_regions_p_1 , textline_mask_tot_ea , num_col_classifier , num_column_is_classified , erosion_hurts , img_bin_light ) :
#print(text_regions_p_1.shape, 'text_regions_p_1 shape run graphics')
#print(erosion_hurts, 'erosion_hurts')
t_in_gr = time . time ( )
@ -3668,7 +3716,6 @@ class Eynollah:
img_poly [ text_regions_p [ : , : ] == 3 ] = 4
img_poly [ text_regions_p [ : , : ] == 6 ] = 5
model_ro_machine , _ = self . start_new_session_and_model ( self . model_reading_order_machine_dir )
height1 = 672 #448
@ -3684,7 +3731,6 @@ class Eynollah:
if contours_only_text_parent_h :
_ , cy_main , x_min_main , x_max_main , y_min_main , y_max_main , _ = find_new_features_of_contours ( contours_only_text_parent_h )
for j in range ( len ( cy_main ) ) :
img_header_and_sep [ int ( y_max_main [ j ] ) : int ( y_max_main [ j ] ) + 12 , int ( x_min_main [ j ] ) : int ( x_max_main [ j ] ) ] = 1
@ -4806,6 +4852,23 @@ class Eynollah:
self . reset_file_name_dir ( os . path . join ( self . dir_in , img_name ) )
#print("text region early -11 in %.1fs", time.time() - t0)
if self . extract_only_images :
img_res , is_image_enhanced , num_col_classifier , num_column_is_classified = self . run_enhancement ( self . light_version )
self . logger . info ( " Enhancing took %.1f s " , time . time ( ) - t0 )
text_regions_p_1 , erosion_hurts , polygons_lines_xml , polygons_of_images , image_page , page_coord , cont_page = self . get_regions_light_v_extract_only_images ( img_res , is_image_enhanced , num_col_classifier )
ocr_all_textlines = None
pcgts = self . writer . build_pagexml_no_full_layout ( [ ] , page_coord , [ ] , [ ] , [ ] , [ ] , polygons_of_images , [ ] , [ ] , [ ] , [ ] , [ ] , cont_page , [ ] , [ ] , ocr_all_textlines )
if self . plotter :
self . plotter . write_images_into_directory ( polygons_of_images , image_page )
if self . dir_in :
self . writer . write_pagexml ( pcgts )
else :
return pcgts
else :
img_res , is_image_enhanced , num_col_classifier , num_column_is_classified = self . run_enhancement ( self . light_version )
self . logger . info ( " Enhancing took %.1f s " , time . time ( ) - t0 )
#print("text region early -1 in %.1fs", time.time() - t0)
vahidrezanezhad
2 months ago
committed by
GitHub