mirror of
https://github.com/qurator-spk/eynollah.git
synced 2025-06-09 20:29:55 +02:00
tables are integrated
This commit is contained in:
parent
80b17af40c
commit
a5c940705a
6 changed files with 502 additions and 33 deletions
|
@ -73,6 +73,12 @@ from qurator.eynollah.eynollah import Eynollah
|
||||||
is_flag=True,
|
is_flag=True,
|
||||||
help="if this parameter set to true, this tool will try to return all elements of layout.",
|
help="if this parameter set to true, this tool will try to return all elements of layout.",
|
||||||
)
|
)
|
||||||
|
@click.option(
|
||||||
|
"--tables/--no-tables",
|
||||||
|
"-tab/-notab",
|
||||||
|
is_flag=True,
|
||||||
|
help="if this parameter set to true, this tool will try to detect tables.",
|
||||||
|
)
|
||||||
@click.option(
|
@click.option(
|
||||||
"--input_binary/--input-RGB",
|
"--input_binary/--input-RGB",
|
||||||
"-ib/-irgb",
|
"-ib/-irgb",
|
||||||
|
@ -109,6 +115,7 @@ def main(
|
||||||
allow_enhancement,
|
allow_enhancement,
|
||||||
curved_line,
|
curved_line,
|
||||||
full_layout,
|
full_layout,
|
||||||
|
tables,
|
||||||
input_binary,
|
input_binary,
|
||||||
allow_scaling,
|
allow_scaling,
|
||||||
headers_off,
|
headers_off,
|
||||||
|
@ -135,6 +142,7 @@ def main(
|
||||||
allow_enhancement=allow_enhancement,
|
allow_enhancement=allow_enhancement,
|
||||||
curved_line=curved_line,
|
curved_line=curved_line,
|
||||||
full_layout=full_layout,
|
full_layout=full_layout,
|
||||||
|
tables=tables,
|
||||||
input_binary=input_binary,
|
input_binary=input_binary,
|
||||||
allow_scaling=allow_scaling,
|
allow_scaling=allow_scaling,
|
||||||
headers_off=headers_off,
|
headers_off=headers_off,
|
||||||
|
|
|
@ -26,12 +26,15 @@ sys.stderr = stderr
|
||||||
import tensorflow as tf
|
import tensorflow as tf
|
||||||
tf.get_logger().setLevel("ERROR")
|
tf.get_logger().setLevel("ERROR")
|
||||||
warnings.filterwarnings("ignore")
|
warnings.filterwarnings("ignore")
|
||||||
|
from scipy.signal import find_peaks
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
from .utils.contour import (
|
from .utils.contour import (
|
||||||
filter_contours_area_of_image,
|
filter_contours_area_of_image,
|
||||||
|
filter_contours_area_of_image_tables,
|
||||||
find_contours_mean_y_diff,
|
find_contours_mean_y_diff,
|
||||||
find_new_features_of_contours,
|
find_new_features_of_contours,
|
||||||
|
find_features_of_contours,
|
||||||
get_text_region_boxes_by_given_contours,
|
get_text_region_boxes_by_given_contours,
|
||||||
get_textregion_contours_in_org_image,
|
get_textregion_contours_in_org_image,
|
||||||
return_contours_of_image,
|
return_contours_of_image,
|
||||||
|
@ -92,6 +95,7 @@ class Eynollah:
|
||||||
allow_enhancement=False,
|
allow_enhancement=False,
|
||||||
curved_line=False,
|
curved_line=False,
|
||||||
full_layout=False,
|
full_layout=False,
|
||||||
|
tables=False,
|
||||||
input_binary=False,
|
input_binary=False,
|
||||||
allow_scaling=False,
|
allow_scaling=False,
|
||||||
headers_off=False,
|
headers_off=False,
|
||||||
|
@ -110,6 +114,7 @@ class Eynollah:
|
||||||
self.allow_enhancement = allow_enhancement
|
self.allow_enhancement = allow_enhancement
|
||||||
self.curved_line = curved_line
|
self.curved_line = curved_line
|
||||||
self.full_layout = full_layout
|
self.full_layout = full_layout
|
||||||
|
self.tables = tables
|
||||||
self.input_binary = input_binary
|
self.input_binary = input_binary
|
||||||
self.allow_scaling = allow_scaling
|
self.allow_scaling = allow_scaling
|
||||||
self.headers_off = headers_off
|
self.headers_off = headers_off
|
||||||
|
@ -137,6 +142,7 @@ class Eynollah:
|
||||||
self.model_page_dir = dir_models + "/model_page_mixed_best.h5"
|
self.model_page_dir = dir_models + "/model_page_mixed_best.h5"
|
||||||
self.model_region_dir_p_ens = dir_models + "/model_ensemble_s.h5"
|
self.model_region_dir_p_ens = dir_models + "/model_ensemble_s.h5"
|
||||||
self.model_textline_dir = dir_models + "/model_textline_newspapers.h5"
|
self.model_textline_dir = dir_models + "/model_textline_newspapers.h5"
|
||||||
|
self.model_tables = dir_models + "/model_tables_ens_mixed_new_2.h5"
|
||||||
|
|
||||||
def _cache_images(self, image_filename=None, image_pil=None):
|
def _cache_images(self, image_filename=None, image_pil=None):
|
||||||
ret = {}
|
ret = {}
|
||||||
|
@ -1612,12 +1618,310 @@ class Eynollah:
|
||||||
order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0])
|
order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0])
|
||||||
|
|
||||||
return order_text_new, id_of_texts_tot
|
return order_text_new, id_of_texts_tot
|
||||||
|
def check_iou_of_bounding_box_and_contour_for_tables(self, layout, table_prediction_early, pixel_tabel, num_col_classifier):
|
||||||
|
layout_org = np.copy(layout)
|
||||||
|
layout_org[:,:,0][layout_org[:,:,0]==pixel_tabel] = 0
|
||||||
|
layout = (layout[:,:,0]==pixel_tabel)*1
|
||||||
|
|
||||||
|
layout =np.repeat(layout[:, :, np.newaxis], 3, axis=2)
|
||||||
|
layout = layout.astype(np.uint8)
|
||||||
|
imgray = cv2.cvtColor(layout, cv2.COLOR_BGR2GRAY )
|
||||||
|
_, thresh = cv2.threshold(imgray, 0, 255, 0)
|
||||||
|
|
||||||
|
contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
|
||||||
|
cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))])
|
||||||
|
|
||||||
|
contours_new = []
|
||||||
|
for i in range(len(contours)):
|
||||||
|
x, y, w, h = cv2.boundingRect(contours[i])
|
||||||
|
iou = cnt_size[i] /float(w*h) *100
|
||||||
|
|
||||||
|
if iou<60:
|
||||||
|
layout_contour = np.zeros((layout_org.shape[0], layout_org.shape[1]))
|
||||||
|
layout_contour= cv2.fillPoly(layout_contour,pts=[contours[i]] ,color=(1,1,1))
|
||||||
|
|
||||||
|
|
||||||
|
layout_contour_sum = layout_contour.sum(axis=0)
|
||||||
|
layout_contour_sum_diff = np.diff(layout_contour_sum)
|
||||||
|
layout_contour_sum_diff= np.abs(layout_contour_sum_diff)
|
||||||
|
layout_contour_sum_diff_smoothed= gaussian_filter1d(layout_contour_sum_diff, 10)
|
||||||
|
|
||||||
|
peaks, _ = find_peaks(layout_contour_sum_diff_smoothed, height=0)
|
||||||
|
peaks= peaks[layout_contour_sum_diff_smoothed[peaks]>4]
|
||||||
|
|
||||||
|
for j in range(len(peaks)):
|
||||||
|
layout_contour[:,peaks[j]-3+1:peaks[j]+1+3] = 0
|
||||||
|
|
||||||
|
layout_contour=cv2.erode(layout_contour[:,:], KERNEL, iterations=5)
|
||||||
|
layout_contour=cv2.dilate(layout_contour[:,:], KERNEL, iterations=5)
|
||||||
|
|
||||||
|
layout_contour =np.repeat(layout_contour[:, :, np.newaxis], 3, axis=2)
|
||||||
|
layout_contour = layout_contour.astype(np.uint8)
|
||||||
|
|
||||||
|
imgray = cv2.cvtColor(layout_contour, cv2.COLOR_BGR2GRAY )
|
||||||
|
_, thresh = cv2.threshold(imgray, 0, 255, 0)
|
||||||
|
|
||||||
|
contours_sep, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
|
||||||
|
|
||||||
|
for ji in range(len(contours_sep) ):
|
||||||
|
contours_new.append(contours_sep[ji])
|
||||||
|
if num_col_classifier>=2:
|
||||||
|
only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1]))
|
||||||
|
only_recent_contour_image= cv2.fillPoly(only_recent_contour_image,pts=[contours_sep[ji]] ,color=(1,1,1))
|
||||||
|
table_pixels_masked_from_early_pre = only_recent_contour_image[:,:]*table_prediction_early[:,:]
|
||||||
|
iou_in = table_pixels_masked_from_early_pre.sum() /float(only_recent_contour_image.sum()) *100
|
||||||
|
|
||||||
|
if iou_in>20:
|
||||||
|
layout_org= cv2.fillPoly(layout_org,pts=[contours_sep[ji]] ,color=(pixel_tabel,pixel_tabel,pixel_tabel))
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
|
||||||
|
layout_org= cv2.fillPoly(layout_org,pts=[contours_sep[ji]] ,color=(pixel_tabel,pixel_tabel,pixel_tabel))
|
||||||
|
|
||||||
|
else:
|
||||||
|
contours_new.append(contours[i])
|
||||||
|
if num_col_classifier>=2:
|
||||||
|
only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1]))
|
||||||
|
only_recent_contour_image= cv2.fillPoly(only_recent_contour_image,pts=[contours[i]] ,color=(1,1,1))
|
||||||
|
|
||||||
|
table_pixels_masked_from_early_pre = only_recent_contour_image[:,:]*table_prediction_early[:,:]
|
||||||
|
iou_in = table_pixels_masked_from_early_pre.sum() /float(only_recent_contour_image.sum()) *100
|
||||||
|
|
||||||
|
if iou_in>20:
|
||||||
|
layout_org= cv2.fillPoly(layout_org,pts=[contours[i]] ,color=(pixel_tabel,pixel_tabel,pixel_tabel))
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
layout_org= cv2.fillPoly(layout_org,pts=[contours[i]] ,color=(pixel_tabel,pixel_tabel,pixel_tabel))
|
||||||
|
|
||||||
|
return layout_org, contours_new
|
||||||
|
def delete_separator_around(self,spliter_y,peaks_neg,image_by_region, pixel_line, pixel_table):
|
||||||
|
# format of subboxes: box=[x1, x2 , y1, y2]
|
||||||
|
pix_del = 100
|
||||||
|
if len(image_by_region.shape)==3:
|
||||||
|
for i in range(len(spliter_y)-1):
|
||||||
|
for j in range(1,len(peaks_neg[i])-1):
|
||||||
|
image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0]==pixel_line ]=0
|
||||||
|
image_by_region[spliter_y[i]:spliter_y[i+1],peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,1]==pixel_line ]=0
|
||||||
|
image_by_region[spliter_y[i]:spliter_y[i+1],peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,2]==pixel_line ]=0
|
||||||
|
|
||||||
|
image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0]==pixel_table ]=0
|
||||||
|
image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,1]==pixel_table ]=0
|
||||||
|
image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,0][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del,2]==pixel_table ]=0
|
||||||
|
else:
|
||||||
|
for i in range(len(spliter_y)-1):
|
||||||
|
for j in range(1,len(peaks_neg[i])-1):
|
||||||
|
image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del]==pixel_line ]=0
|
||||||
|
|
||||||
|
image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del][image_by_region[int(spliter_y[i]):int(spliter_y[i+1]),peaks_neg[i][j]-pix_del:peaks_neg[i][j]+pix_del]==pixel_table ]=0
|
||||||
|
return image_by_region
|
||||||
|
def add_tables_heuristic_to_layout(self, image_regions_eraly_p,boxes, slope_mean_hor, spliter_y,peaks_neg_tot, image_revised, num_col_classifier, min_area, pixel_line):
|
||||||
|
pixel_table =10
|
||||||
|
image_revised_1 = self.delete_separator_around(spliter_y, peaks_neg_tot, image_revised, pixel_line, pixel_table)
|
||||||
|
img_comm_e = np.zeros(image_revised_1.shape)
|
||||||
|
img_comm = np.repeat(img_comm_e[:, :, np.newaxis], 3, axis=2)
|
||||||
|
|
||||||
|
for indiv in np.unique(image_revised_1):
|
||||||
|
image_col=(image_revised_1==indiv)*255
|
||||||
|
img_comm_in=np.repeat(image_col[:, :, np.newaxis], 3, axis=2)
|
||||||
|
img_comm_in=img_comm_in.astype(np.uint8)
|
||||||
|
|
||||||
|
imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY)
|
||||||
|
ret, thresh = cv2.threshold(imgray, 0, 255, 0)
|
||||||
|
contours,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
|
||||||
|
|
||||||
|
if indiv==pixel_table:
|
||||||
|
main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area = 1, min_area = 0.001)
|
||||||
|
else:
|
||||||
|
main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area = 1, min_area = min_area)
|
||||||
|
|
||||||
|
img_comm = cv2.fillPoly(img_comm, pts = main_contours, color = (indiv, indiv, indiv))
|
||||||
|
img_comm = img_comm.astype(np.uint8)
|
||||||
|
|
||||||
|
if not self.isNaN(slope_mean_hor):
|
||||||
|
image_revised_last = np.zeros((image_regions_eraly_p.shape[0], image_regions_eraly_p.shape[1],3))
|
||||||
|
for i in range(len(boxes)):
|
||||||
|
image_box=img_comm[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:]
|
||||||
|
try:
|
||||||
|
image_box_tabels_1=(image_box[:,:,0]==pixel_table)*1
|
||||||
|
contours_tab,_=return_contours_of_image(image_box_tabels_1)
|
||||||
|
contours_tab=filter_contours_area_of_image_tables(image_box_tabels_1,contours_tab,_,1,0.003)
|
||||||
|
image_box_tabels_1=(image_box[:,:,0]==pixel_line)*1
|
||||||
|
|
||||||
|
image_box_tabels_and_m_text=( (image_box[:,:,0]==pixel_table) | (image_box[:,:,0]==1) )*1
|
||||||
|
image_box_tabels_and_m_text=image_box_tabels_and_m_text.astype(np.uint8)
|
||||||
|
|
||||||
|
image_box_tabels_1=image_box_tabels_1.astype(np.uint8)
|
||||||
|
image_box_tabels_1 = cv2.dilate(image_box_tabels_1,KERNEL,iterations = 5)
|
||||||
|
|
||||||
|
contours_table_m_text,_=return_contours_of_image(image_box_tabels_and_m_text)
|
||||||
|
image_box_tabels=np.repeat(image_box_tabels_1[:, :, np.newaxis], 3, axis=2)
|
||||||
|
|
||||||
|
image_box_tabels=image_box_tabels.astype(np.uint8)
|
||||||
|
imgray = cv2.cvtColor(image_box_tabels, cv2.COLOR_BGR2GRAY)
|
||||||
|
ret, thresh = cv2.threshold(imgray, 0, 255, 0)
|
||||||
|
|
||||||
|
contours_line,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
|
||||||
|
|
||||||
|
y_min_main_line ,y_max_main_line=find_features_of_contours(contours_line)
|
||||||
|
y_min_main_tab ,y_max_main_tab=find_features_of_contours(contours_tab)
|
||||||
|
|
||||||
|
cx_tab_m_text,cy_tab_m_text ,x_min_tab_m_text , x_max_tab_m_text, y_min_tab_m_text ,y_max_tab_m_text, _= find_new_features_of_contours(contours_table_m_text)
|
||||||
|
cx_tabl,cy_tabl ,x_min_tabl , x_max_tabl, y_min_tabl ,y_max_tabl,_= find_new_features_of_contours(contours_tab)
|
||||||
|
|
||||||
|
if len(y_min_main_tab )>0:
|
||||||
|
y_down_tabs=[]
|
||||||
|
y_up_tabs=[]
|
||||||
|
|
||||||
|
for i_t in range(len(y_min_main_tab )):
|
||||||
|
y_down_tab=[]
|
||||||
|
y_up_tab=[]
|
||||||
|
for i_l in range(len(y_min_main_line)):
|
||||||
|
if y_min_main_tab[i_t]>y_min_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l] and y_min_main_tab[i_t]>y_max_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l]:
|
||||||
|
pass
|
||||||
|
elif y_min_main_tab[i_t]<y_max_main_line[i_l] and y_max_main_tab[i_t]<y_max_main_line[i_l] and y_max_main_tab[i_t]<y_min_main_line[i_l] and y_min_main_tab[i_t]<y_min_main_line[i_l]:
|
||||||
|
pass
|
||||||
|
elif np.abs(y_max_main_line[i_l]-y_min_main_line[i_l])<100:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
y_up_tab.append(np.min([y_min_main_line[i_l], y_min_main_tab[i_t] ]) )
|
||||||
|
y_down_tab.append( np.max([ y_max_main_line[i_l],y_max_main_tab[i_t] ]) )
|
||||||
|
|
||||||
|
if len(y_up_tab)==0:
|
||||||
|
y_up_tabs.append(y_min_main_tab[i_t])
|
||||||
|
y_down_tabs.append(y_max_main_tab[i_t])
|
||||||
|
else:
|
||||||
|
y_up_tabs.append(np.min(y_up_tab))
|
||||||
|
y_down_tabs.append(np.max(y_down_tab))
|
||||||
|
else:
|
||||||
|
y_down_tabs=[]
|
||||||
|
y_up_tabs=[]
|
||||||
|
pass
|
||||||
|
except:
|
||||||
|
y_down_tabs=[]
|
||||||
|
y_up_tabs=[]
|
||||||
|
|
||||||
|
for ii in range(len(y_up_tabs)):
|
||||||
|
image_box[y_up_tabs[ii]:y_down_tabs[ii],:,0]=pixel_table
|
||||||
|
|
||||||
|
image_revised_last[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:]=image_box[:,:,:]
|
||||||
|
else:
|
||||||
|
for i in range(len(boxes)):
|
||||||
|
|
||||||
|
image_box=img_comm[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:]
|
||||||
|
image_revised_last[int(boxes[i][2]):int(boxes[i][3]),int(boxes[i][0]):int(boxes[i][1]),:]=image_box[:,:,:]
|
||||||
|
|
||||||
|
if num_col_classifier==1:
|
||||||
|
img_tables_col_1=( image_revised_last[:,:,0]==pixel_table )*1
|
||||||
|
img_tables_col_1=img_tables_col_1.astype(np.uint8)
|
||||||
|
contours_table_col1,_=return_contours_of_image(img_tables_col_1)
|
||||||
|
|
||||||
|
_,_ ,_ , _, y_min_tab_col1 ,y_max_tab_col1, _= find_new_features_of_contours(contours_table_col1)
|
||||||
|
|
||||||
|
if len(y_min_tab_col1)>0:
|
||||||
|
for ijv in range(len(y_min_tab_col1)):
|
||||||
|
image_revised_last[int(y_min_tab_col1[ijv]):int(y_max_tab_col1[ijv]),:,:]=pixel_table
|
||||||
|
return image_revised_last
|
||||||
def do_order_of_regions(self, *args, **kwargs):
|
def do_order_of_regions(self, *args, **kwargs):
|
||||||
if self.full_layout:
|
if self.full_layout:
|
||||||
return self.do_order_of_regions_full_layout(*args, **kwargs)
|
return self.do_order_of_regions_full_layout(*args, **kwargs)
|
||||||
return self.do_order_of_regions_no_full_layout(*args, **kwargs)
|
return self.do_order_of_regions_no_full_layout(*args, **kwargs)
|
||||||
|
|
||||||
|
def get_tables_from_model(self, img, num_col_classifier):
|
||||||
|
img_org = np.copy(img)
|
||||||
|
|
||||||
|
img_height_h = img_org.shape[0]
|
||||||
|
img_width_h = img_org.shape[1]
|
||||||
|
|
||||||
|
model_region, session_region = self.start_new_session_and_model(self.model_tables)
|
||||||
|
|
||||||
|
patches = False
|
||||||
|
|
||||||
|
if num_col_classifier < 4 and num_col_classifier > 2:
|
||||||
|
prediction_table = self.do_prediction(patches, img, model_region)
|
||||||
|
elif num_col_classifier ==2:
|
||||||
|
height_ext = 0#int( img.shape[0]/4. )
|
||||||
|
h_start = int(height_ext/2.)
|
||||||
|
width_ext = int( img.shape[1]/8. )
|
||||||
|
w_start = int(width_ext/2.)
|
||||||
|
|
||||||
|
height_new = img.shape[0]+height_ext
|
||||||
|
width_new = img.shape[1]+width_ext
|
||||||
|
|
||||||
|
img_new =np.ones((height_new,width_new,img.shape[2])).astype(float)*0
|
||||||
|
img_new[h_start:h_start+img.shape[0] ,w_start: w_start+img.shape[1], : ] =img[:,:,:]
|
||||||
|
|
||||||
|
prediction_ext = self.do_prediction(patches, img_new, model_region)
|
||||||
|
prediction_table = prediction_ext[h_start:h_start+img.shape[0] ,w_start: w_start+img.shape[1], : ]
|
||||||
|
|
||||||
|
prediction_table = prediction_table.astype(np.int16)
|
||||||
|
|
||||||
|
elif num_col_classifier ==1:
|
||||||
|
height_ext = 0# int( img.shape[0]/4. )
|
||||||
|
h_start = int(height_ext/2.)
|
||||||
|
width_ext = int( img.shape[1]/4. )
|
||||||
|
w_start = int(width_ext/2.)
|
||||||
|
|
||||||
|
height_new = img.shape[0]+height_ext
|
||||||
|
width_new = img.shape[1]+width_ext
|
||||||
|
|
||||||
|
img_new =np.ones((height_new,width_new,img.shape[2])).astype(float)*0
|
||||||
|
img_new[h_start:h_start+img.shape[0] ,w_start: w_start+img.shape[1], : ] =img[:,:,:]
|
||||||
|
|
||||||
|
prediction_ext = self.do_prediction(patches, img_new, model_region)
|
||||||
|
prediction_table = prediction_ext[h_start:h_start+img.shape[0] ,w_start: w_start+img.shape[1], : ]
|
||||||
|
prediction_table = prediction_table.astype(np.int16)
|
||||||
|
|
||||||
|
elif num_col_classifier ==60:
|
||||||
|
prediction_table = np.zeros(img.shape)
|
||||||
|
img_w_half = int(img.shape[1]/2.)
|
||||||
|
img_h_half = int(img.shape[0]/2.)
|
||||||
|
|
||||||
|
pre1 = self.do_prediction(patches, img[0:img_h_half,0:img_w_half,:], model_region)
|
||||||
|
pre2 = self.do_prediction(patches, img[0:img_h_half,img_w_half:,:], model_region)
|
||||||
|
|
||||||
|
pre3 = self.do_prediction(patches, img[img_h_half:,0:img_w_half,:], model_region)
|
||||||
|
pre4 = self.do_prediction(patches, img[img_h_half:,img_w_half:,:], model_region)
|
||||||
|
|
||||||
|
prediction_table[0:img_h_half,0:img_w_half,:] = pre1[:,:,:]
|
||||||
|
prediction_table[0:img_h_half,img_w_half:,:] = pre2[:,:,:]
|
||||||
|
|
||||||
|
prediction_table[img_h_half:,0:img_w_half,:] = pre3[:,:,:]
|
||||||
|
prediction_table[img_h_half:,img_w_half:,:] = pre4[:,:,:]
|
||||||
|
prediction_table = prediction_table.astype(np.int16)
|
||||||
|
else:
|
||||||
|
prediction_table = np.zeros(img.shape)
|
||||||
|
img_w_half = int(img.shape[1]/2.)
|
||||||
|
|
||||||
|
pre1 = self.do_prediction(patches, img[:,0:img_w_half,:], model_region)
|
||||||
|
pre2 = self.do_prediction(patches, img[:,img_w_half:,:], model_region)
|
||||||
|
|
||||||
|
pre_full = self.do_prediction(patches, img[:,:,:], model_region)
|
||||||
|
|
||||||
|
|
||||||
|
prediction_table_full_erode = cv2.erode(pre_full[:,:,0], KERNEL, iterations=4)
|
||||||
|
prediction_table_full_erode = cv2.dilate(prediction_table_full_erode, KERNEL, iterations=4)
|
||||||
|
|
||||||
|
prediction_table[:,0:img_w_half,:] = pre1[:,:,:]
|
||||||
|
prediction_table[:,img_w_half:,:] = pre2[:,:,:]
|
||||||
|
|
||||||
|
prediction_table[:,:,0][prediction_table_full_erode[:,:]==1]=1
|
||||||
|
prediction_table = prediction_table.astype(np.int16)
|
||||||
|
|
||||||
|
#prediction_table_erode = cv2.erode(prediction_table[:,:,0], self.kernel, iterations=6)
|
||||||
|
#prediction_table_erode = cv2.dilate(prediction_table_erode, self.kernel, iterations=6)
|
||||||
|
|
||||||
|
prediction_table_erode = cv2.erode(prediction_table[:,:,0], KERNEL, iterations=20)
|
||||||
|
prediction_table_erode = cv2.dilate(prediction_table_erode, KERNEL, iterations=20)
|
||||||
|
|
||||||
|
del model_region
|
||||||
|
del session_region
|
||||||
|
gc.collect()
|
||||||
|
|
||||||
|
|
||||||
|
return prediction_table_erode.astype(np.int16)
|
||||||
|
|
||||||
def run_graphics_and_columns(self, text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts):
|
def run_graphics_and_columns(self, text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts):
|
||||||
img_g = self.imread(grayscale=True, uint8=True)
|
img_g = self.imread(grayscale=True, uint8=True)
|
||||||
|
|
||||||
|
@ -1628,6 +1932,12 @@ class Eynollah:
|
||||||
img_g3[:, :, 2] = img_g[:, :]
|
img_g3[:, :, 2] = img_g[:, :]
|
||||||
|
|
||||||
image_page, page_coord, cont_page = self.extract_page()
|
image_page, page_coord, cont_page = self.extract_page()
|
||||||
|
|
||||||
|
if self.tables:
|
||||||
|
table_prediction = self.get_tables_from_model(image_page, num_col_classifier)
|
||||||
|
else:
|
||||||
|
table_prediction = (np.zeros((image_page.shape[0], image_page.shape[1]))).astype(np.int16)
|
||||||
|
|
||||||
if self.plotter:
|
if self.plotter:
|
||||||
self.plotter.save_page_image(image_page)
|
self.plotter.save_page_image(image_page)
|
||||||
|
|
||||||
|
@ -1655,7 +1965,7 @@ class Eynollah:
|
||||||
except Exception as why:
|
except Exception as why:
|
||||||
self.logger.error(why)
|
self.logger.error(why)
|
||||||
num_col = None
|
num_col = None
|
||||||
return num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, text_regions_p_1, cont_page
|
return num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, text_regions_p_1, cont_page, table_prediction
|
||||||
|
|
||||||
def run_enhancement(self):
|
def run_enhancement(self):
|
||||||
self.logger.info("resize and enhance image")
|
self.logger.info("resize and enhance image")
|
||||||
|
@ -1699,7 +2009,7 @@ class Eynollah:
|
||||||
self.logger.info("slope_deskew: %s", slope_deskew)
|
self.logger.info("slope_deskew: %s", slope_deskew)
|
||||||
return slope_deskew, slope_first
|
return slope_deskew, slope_first
|
||||||
|
|
||||||
def run_marginals(self, image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1):
|
def run_marginals(self, image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction):
|
||||||
image_page_rotated, textline_mask_tot = image_page[:, :], textline_mask_tot_ea[:, :]
|
image_page_rotated, textline_mask_tot = image_page[:, :], textline_mask_tot_ea[:, :]
|
||||||
textline_mask_tot[mask_images[:, :] == 1] = 0
|
textline_mask_tot[mask_images[:, :] == 1] = 0
|
||||||
|
|
||||||
|
@ -1710,6 +2020,8 @@ class Eynollah:
|
||||||
if num_col_classifier in (1, 2):
|
if num_col_classifier in (1, 2):
|
||||||
try:
|
try:
|
||||||
regions_without_separators = (text_regions_p[:, :] == 1) * 1
|
regions_without_separators = (text_regions_p[:, :] == 1) * 1
|
||||||
|
if self.tables:
|
||||||
|
regions_without_separators[table_prediction==1] = 1
|
||||||
regions_without_separators = regions_without_separators.astype(np.uint8)
|
regions_without_separators = regions_without_separators.astype(np.uint8)
|
||||||
text_regions_p = get_marginals(rotate_image(regions_without_separators, slope_deskew), text_regions_p, num_col_classifier, slope_deskew, kernel=KERNEL)
|
text_regions_p = get_marginals(rotate_image(regions_without_separators, slope_deskew), text_regions_p, num_col_classifier, slope_deskew, kernel=KERNEL)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -1720,14 +2032,19 @@ class Eynollah:
|
||||||
self.plotter.save_plot_of_layout_main(text_regions_p, image_page)
|
self.plotter.save_plot_of_layout_main(text_regions_p, image_page)
|
||||||
return textline_mask_tot, text_regions_p, image_page_rotated
|
return textline_mask_tot, text_regions_p, image_page_rotated
|
||||||
|
|
||||||
def run_boxes_no_full_layout(self, image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, erosion_hurts):
|
def run_boxes_no_full_layout(self, image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, table_prediction, erosion_hurts):
|
||||||
self.logger.debug('enter run_boxes_no_full_layout')
|
self.logger.debug('enter run_boxes_no_full_layout')
|
||||||
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
||||||
_, textline_mask_tot_d, text_regions_p_1_n = rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, slope_deskew)
|
_, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew)
|
||||||
text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1])
|
text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1])
|
||||||
textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1])
|
textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1])
|
||||||
|
table_prediction_n = resize_image(table_prediction_n, text_regions_p.shape[0], text_regions_p.shape[1])
|
||||||
regions_without_separators_d = (text_regions_p_1_n[:, :] == 1) * 1
|
regions_without_separators_d = (text_regions_p_1_n[:, :] == 1) * 1
|
||||||
|
if self.tables:
|
||||||
|
regions_without_separators_d[table_prediction_n[:,:] == 1] = 1
|
||||||
regions_without_separators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions)
|
regions_without_separators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions)
|
||||||
|
if self.tables:
|
||||||
|
regions_without_separators[table_prediction ==1 ] = 1
|
||||||
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
text_regions_p_1_n = None
|
text_regions_p_1_n = None
|
||||||
textline_mask_tot_d = None
|
textline_mask_tot_d = None
|
||||||
|
@ -1751,26 +2068,148 @@ class Eynollah:
|
||||||
regions_without_separators_d = cv2.erode(regions_without_separators_d[:, :], KERNEL, iterations=6)
|
regions_without_separators_d = cv2.erode(regions_without_separators_d[:, :], KERNEL, iterations=6)
|
||||||
t1 = time.time()
|
t1 = time.time()
|
||||||
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
boxes = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts)
|
boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts)
|
||||||
boxes_d = None
|
boxes_d = None
|
||||||
self.logger.debug("len(boxes): %s", len(boxes))
|
self.logger.debug("len(boxes): %s", len(boxes))
|
||||||
|
|
||||||
|
text_regions_p_tables = np.copy(text_regions_p)
|
||||||
|
text_regions_p_tables[:,:][(table_prediction[:,:] == 1)] = 10
|
||||||
|
pixel_line = 3
|
||||||
|
img_revised_tab2 = self.add_tables_heuristic_to_layout(text_regions_p_tables, boxes, 0, splitter_y_new, peaks_neg_tot_tables, text_regions_p_tables , num_col_classifier , 0.000005, pixel_line)
|
||||||
|
img_revised_tab2, contoures_tables = self.check_iou_of_bounding_box_and_contour_for_tables(img_revised_tab2,table_prediction, 10, num_col_classifier)
|
||||||
else:
|
else:
|
||||||
boxes_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts)
|
boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts)
|
||||||
boxes = None
|
boxes = None
|
||||||
self.logger.debug("len(boxes): %s", len(boxes_d))
|
self.logger.debug("len(boxes): %s", len(boxes_d))
|
||||||
|
|
||||||
|
text_regions_p_tables = np.copy(text_regions_p_1_n)
|
||||||
|
text_regions_p_tables =np.round(text_regions_p_tables)
|
||||||
|
text_regions_p_tables[:,:][(text_regions_p_tables[:,:] != 3) & (table_prediction_n[:,:] == 1)] = 10
|
||||||
|
|
||||||
|
pixel_line = 3
|
||||||
|
img_revised_tab2 = self.add_tables_heuristic_to_layout(text_regions_p_tables,boxes_d,0,splitter_y_new_d,peaks_neg_tot_tables_d,text_regions_p_tables, num_col_classifier, 0.000005, pixel_line)
|
||||||
|
img_revised_tab2_d,_ = self.check_iou_of_bounding_box_and_contour_for_tables(img_revised_tab2,table_prediction_n, 10, num_col_classifier)
|
||||||
|
|
||||||
|
img_revised_tab2_d_rotated = rotate_image(img_revised_tab2_d, -slope_deskew)
|
||||||
|
img_revised_tab2_d_rotated = np.round(img_revised_tab2_d_rotated)
|
||||||
|
img_revised_tab2_d_rotated = img_revised_tab2_d_rotated.astype(np.int8)
|
||||||
|
img_revised_tab2_d_rotated = resize_image(img_revised_tab2_d_rotated, text_regions_p.shape[0], text_regions_p.shape[1])
|
||||||
|
|
||||||
self.logger.info("detecting boxes took %ss", str(time.time() - t1))
|
self.logger.info("detecting boxes took %ss", str(time.time() - t1))
|
||||||
|
|
||||||
|
if self.tables:
|
||||||
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
|
img_revised_tab = np.copy(img_revised_tab2[:,:,0])
|
||||||
|
else:
|
||||||
|
img_revised_tab = np.copy(text_regions_p[:,:])
|
||||||
|
img_revised_tab[:,:][img_revised_tab[:,:] == 10] = 0
|
||||||
|
img_revised_tab[:,:][img_revised_tab2_d_rotated[:,:,0] == 10] = 10
|
||||||
|
|
||||||
|
text_regions_p[:,:][text_regions_p[:,:]==10] = 0
|
||||||
|
text_regions_p[:,:][img_revised_tab[:,:]==10] = 10
|
||||||
|
else:
|
||||||
img_revised_tab=text_regions_p[:,:]
|
img_revised_tab=text_regions_p[:,:]
|
||||||
|
#img_revised_tab = text_regions_p[:, :]
|
||||||
polygons_of_images = return_contours_of_interested_region(img_revised_tab, 2)
|
polygons_of_images = return_contours_of_interested_region(img_revised_tab, 2)
|
||||||
|
|
||||||
# plt.imshow(img_revised_tab)
|
pixel_img = 4
|
||||||
# plt.show()
|
min_area_mar = 0.00001
|
||||||
|
polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar)
|
||||||
|
|
||||||
|
pixel_img = 10
|
||||||
|
contours_tables = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar)
|
||||||
|
|
||||||
|
|
||||||
K.clear_session()
|
K.clear_session()
|
||||||
self.logger.debug('exit run_boxes_no_full_layout')
|
self.logger.debug('exit run_boxes_no_full_layout')
|
||||||
return polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, boxes, boxes_d
|
return polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, boxes, boxes_d, polygons_of_marginals, contours_tables
|
||||||
|
|
||||||
def run_boxes_full_layout(self, image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions):
|
def run_boxes_full_layout(self, image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions, table_prediction, erosion_hurts):
|
||||||
self.logger.debug('enter run_boxes_full_layout')
|
self.logger.debug('enter run_boxes_full_layout')
|
||||||
|
|
||||||
|
if self.tables:
|
||||||
|
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
||||||
|
image_page_rotated_n,textline_mask_tot_d,text_regions_p_1_n , table_prediction_n = rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew)
|
||||||
|
|
||||||
|
text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1])
|
||||||
|
textline_mask_tot_d = resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1])
|
||||||
|
table_prediction_n = resize_image(table_prediction_n,text_regions_p.shape[0],text_regions_p.shape[1])
|
||||||
|
|
||||||
|
regions_without_seperators_d=(text_regions_p_1_n[:,:] == 1)*1
|
||||||
|
regions_without_seperators_d[table_prediction_n[:,:] == 1] = 1
|
||||||
|
|
||||||
|
regions_without_seperators = (text_regions_p[:,:] == 1)*1#( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_seperators_new(text_regions_p[:,:,0],img_only_regions)
|
||||||
|
regions_without_seperators[table_prediction == 1] = 1
|
||||||
|
|
||||||
|
pixel_lines=3
|
||||||
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
|
num_col, peaks_neg_fin, matrix_of_lines_ch, splitter_y_new, seperators_closeup_n = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines)
|
||||||
|
|
||||||
|
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
||||||
|
num_col_d, peaks_neg_fin_d, matrix_of_lines_ch_d, splitter_y_new_d, seperators_closeup_n_d = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2),num_col_classifier,pixel_lines)
|
||||||
|
K.clear_session()
|
||||||
|
gc.collect()
|
||||||
|
|
||||||
|
if num_col_classifier>=3:
|
||||||
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
|
regions_without_seperators = regions_without_seperators.astype(np.uint8)
|
||||||
|
regions_without_seperators = cv2.erode(regions_without_seperators[:,:], KERNEL, iterations=6)
|
||||||
|
|
||||||
|
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
||||||
|
regions_without_seperators_d = regions_without_seperators_d.astype(np.uint8)
|
||||||
|
regions_without_seperators_d = cv2.erode(regions_without_seperators_d[:,:], KERNEL, iterations=6)
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
|
||||||
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
|
boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_seperators, matrix_of_lines_ch, num_col_classifier, erosion_hurts)
|
||||||
|
text_regions_p_tables = np.copy(text_regions_p)
|
||||||
|
text_regions_p_tables[:,:][(table_prediction[:,:]==1)] = 10
|
||||||
|
pixel_line = 3
|
||||||
|
img_revised_tab2 = self.add_tables_heuristic_to_layout(text_regions_p_tables, boxes, 0, splitter_y_new, peaks_neg_tot_tables, text_regions_p_tables , num_col_classifier , 0.000005, pixel_line)
|
||||||
|
|
||||||
|
img_revised_tab2,contoures_tables = self.check_iou_of_bounding_box_and_contour_for_tables(img_revised_tab2, table_prediction, 10, num_col_classifier)
|
||||||
|
|
||||||
|
else:
|
||||||
|
boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_seperators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts)
|
||||||
|
text_regions_p_tables = np.copy(text_regions_p_1_n)
|
||||||
|
text_regions_p_tables = np.round(text_regions_p_tables)
|
||||||
|
text_regions_p_tables[:,:][(text_regions_p_tables[:,:]!=3) & (table_prediction_n[:,:]==1)] = 10
|
||||||
|
|
||||||
|
pixel_line = 3
|
||||||
|
img_revised_tab2 = self.add_tables_heuristic_to_layout(text_regions_p_tables,boxes_d,0,splitter_y_new_d,peaks_neg_tot_tables_d,text_regions_p_tables, num_col_classifier, 0.000005, pixel_line)
|
||||||
|
|
||||||
|
img_revised_tab2_d,_ = self.check_iou_of_bounding_box_and_contour_for_tables(img_revised_tab2, table_prediction_n, 10, num_col_classifier)
|
||||||
|
img_revised_tab2_d_rotated = rotate_image(img_revised_tab2_d, -slope_deskew)
|
||||||
|
|
||||||
|
|
||||||
|
img_revised_tab2_d_rotated = np.round(img_revised_tab2_d_rotated)
|
||||||
|
img_revised_tab2_d_rotated = img_revised_tab2_d_rotated.astype(np.int8)
|
||||||
|
|
||||||
|
img_revised_tab2_d_rotated = resize_image(img_revised_tab2_d_rotated, text_regions_p.shape[0], text_regions_p.shape[1])
|
||||||
|
|
||||||
|
|
||||||
|
if np.abs(slope_deskew) < 0.13:
|
||||||
|
img_revised_tab = np.copy(img_revised_tab2[:,:,0])
|
||||||
|
else:
|
||||||
|
img_revised_tab = np.copy(text_regions_p[:,:])
|
||||||
|
img_revised_tab[:,:][img_revised_tab[:,:] == 10] = 0
|
||||||
|
img_revised_tab[:,:][img_revised_tab2_d_rotated[:,:,0] == 10] = 10
|
||||||
|
|
||||||
|
|
||||||
|
##img_revised_tab=img_revised_tab2[:,:,0]
|
||||||
|
#img_revised_tab=text_regions_p[:,:]
|
||||||
|
text_regions_p[:,:][text_regions_p[:,:]==10] = 0
|
||||||
|
text_regions_p[:,:][img_revised_tab[:,:]==10] = 10
|
||||||
|
#img_revised_tab[img_revised_tab2[:,:,0]==10] =10
|
||||||
|
|
||||||
|
pixel_img = 4
|
||||||
|
min_area_mar = 0.00001
|
||||||
|
polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar)
|
||||||
|
|
||||||
|
pixel_img = 10
|
||||||
|
contours_tables = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar)
|
||||||
|
|
||||||
# set first model with second model
|
# set first model with second model
|
||||||
text_regions_p[:, :][text_regions_p[:, :] == 2] = 5
|
text_regions_p[:, :][text_regions_p[:, :] == 2] = 5
|
||||||
text_regions_p[:, :][text_regions_p[:, :] == 3] = 6
|
text_regions_p[:, :][text_regions_p[:, :] == 3] = 6
|
||||||
|
@ -1830,7 +2269,7 @@ class Eynollah:
|
||||||
img_revised_tab = np.copy(text_regions_p[:, :])
|
img_revised_tab = np.copy(text_regions_p[:, :])
|
||||||
polygons_of_images = return_contours_of_interested_region(img_revised_tab, 5)
|
polygons_of_images = return_contours_of_interested_region(img_revised_tab, 5)
|
||||||
self.logger.debug('exit run_boxes_full_layout')
|
self.logger.debug('exit run_boxes_full_layout')
|
||||||
return polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, regions_fully, regions_without_separators
|
return polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, regions_fully, regions_without_separators, polygons_of_marginals, contours_tables
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
"""
|
"""
|
||||||
|
@ -1848,7 +2287,7 @@ class Eynollah:
|
||||||
self.logger.info("Textregion detection took %ss ", str(time.time() - t1))
|
self.logger.info("Textregion detection took %ss ", str(time.time() - t1))
|
||||||
|
|
||||||
t1 = time.time()
|
t1 = time.time()
|
||||||
num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, text_regions_p_1, cont_page = \
|
num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, text_regions_p_1, cont_page, table_prediction = \
|
||||||
self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts)
|
self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts)
|
||||||
self.logger.info("Graphics detection took %ss ", str(time.time() - t1))
|
self.logger.info("Graphics detection took %ss ", str(time.time() - t1))
|
||||||
self.logger.info('cont_page %s', cont_page)
|
self.logger.info('cont_page %s', cont_page)
|
||||||
|
@ -1868,19 +2307,15 @@ class Eynollah:
|
||||||
self.logger.info("deskewing took %ss", str(time.time() - t1))
|
self.logger.info("deskewing took %ss", str(time.time() - t1))
|
||||||
t1 = time.time()
|
t1 = time.time()
|
||||||
|
|
||||||
textline_mask_tot, text_regions_p, image_page_rotated = self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1)
|
textline_mask_tot, text_regions_p, image_page_rotated = self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1, table_prediction)
|
||||||
self.logger.info("detection of marginals took %ss", str(time.time() - t1))
|
self.logger.info("detection of marginals took %ss", str(time.time() - t1))
|
||||||
t1 = time.time()
|
t1 = time.time()
|
||||||
|
|
||||||
if not self.full_layout:
|
if not self.full_layout:
|
||||||
polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, boxes, boxes_d = self.run_boxes_no_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, erosion_hurts)
|
polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, boxes, boxes_d, polygons_of_marginals, contours_tables = self.run_boxes_no_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, table_prediction, erosion_hurts)
|
||||||
|
|
||||||
pixel_img = 4
|
|
||||||
min_area_mar = 0.00001
|
|
||||||
polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar)
|
|
||||||
|
|
||||||
if self.full_layout:
|
if self.full_layout:
|
||||||
polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, regions_fully, regions_without_separators = self.run_boxes_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions)
|
polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, regions_fully, regions_without_separators, polygons_of_marginals, contours_tables = self.run_boxes_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions, table_prediction, erosion_hurts)
|
||||||
|
|
||||||
text_only = ((img_revised_tab[:, :] == 1)) * 1
|
text_only = ((img_revised_tab[:, :] == 1)) * 1
|
||||||
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
if np.abs(slope_deskew) >= SLOPE_THRESHOLD:
|
||||||
|
@ -2018,7 +2453,6 @@ class Eynollah:
|
||||||
|
|
||||||
K.clear_session()
|
K.clear_session()
|
||||||
|
|
||||||
polygons_of_tabels = []
|
|
||||||
pixel_img = 4
|
pixel_img = 4
|
||||||
polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img)
|
polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img)
|
||||||
all_found_texline_polygons = adhere_drop_capital_region_into_corresponding_textline(text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, kernel=KERNEL, curved_line=self.curved_line)
|
all_found_texline_polygons = adhere_drop_capital_region_into_corresponding_textline(text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, kernel=KERNEL, curved_line=self.curved_line)
|
||||||
|
@ -2058,9 +2492,9 @@ class Eynollah:
|
||||||
regions_without_separators_d[(random_pixels_for_image[:, :] == 1) & (text_regions_p_1_n[:, :] == 5)] = 1
|
regions_without_separators_d[(random_pixels_for_image[:, :] == 1) & (text_regions_p_1_n[:, :] == 5)] = 1
|
||||||
|
|
||||||
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
if np.abs(slope_deskew) < SLOPE_THRESHOLD:
|
||||||
boxes = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts)
|
boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts)
|
||||||
else:
|
else:
|
||||||
boxes_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts)
|
boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts)
|
||||||
|
|
||||||
if self.plotter:
|
if self.plotter:
|
||||||
self.plotter.write_images_into_directory(polygons_of_images, image_page)
|
self.plotter.write_images_into_directory(polygons_of_images, image_page)
|
||||||
|
@ -2071,7 +2505,7 @@ class Eynollah:
|
||||||
else:
|
else:
|
||||||
order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d)
|
order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d)
|
||||||
|
|
||||||
pcgts = self.writer.build_pagexml_full_layout(contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_found_texline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, polygons_of_tabels, polygons_of_drop_capitals, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_xml)
|
pcgts = self.writer.build_pagexml_full_layout(contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_found_texline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_xml)
|
||||||
self.logger.info("Job done in %ss", str(time.time() - t0))
|
self.logger.info("Job done in %ss", str(time.time() - t0))
|
||||||
return pcgts
|
return pcgts
|
||||||
else:
|
else:
|
||||||
|
@ -2081,6 +2515,6 @@ class Eynollah:
|
||||||
else:
|
else:
|
||||||
contours_only_text_parent_d_ordered = list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con])
|
contours_only_text_parent_d_ordered = list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con])
|
||||||
order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d)
|
order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d)
|
||||||
pcgts = self.writer.build_pagexml_no_full_layout(txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml)
|
pcgts = self.writer.build_pagexml_no_full_layout(txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml, contours_tables)
|
||||||
self.logger.info("Job done in %ss", str(time.time() - t0))
|
self.logger.info("Job done in %ss", str(time.time() - t0))
|
||||||
return pcgts
|
return pcgts
|
||||||
|
|
|
@ -1585,7 +1585,7 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, pixel_l
|
||||||
|
|
||||||
def return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts):
|
def return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts):
|
||||||
boxes=[]
|
boxes=[]
|
||||||
|
peaks_neg_tot_tables = []
|
||||||
|
|
||||||
for i in range(len(splitter_y_new)-1):
|
for i in range(len(splitter_y_new)-1):
|
||||||
#print(splitter_y_new[i],splitter_y_new[i+1])
|
#print(splitter_y_new[i],splitter_y_new[i+1])
|
||||||
|
@ -1679,6 +1679,8 @@ def return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_witho
|
||||||
|
|
||||||
peaks_neg_tot=return_points_with_boundies(peaks_neg_fin,0, regions_without_separators[:,:].shape[1])
|
peaks_neg_tot=return_points_with_boundies(peaks_neg_fin,0, regions_without_separators[:,:].shape[1])
|
||||||
|
|
||||||
|
peaks_neg_tot_tables.append(peaks_neg_tot)
|
||||||
|
|
||||||
reading_order_type,x_starting,x_ending,y_type_2,y_diff_type_2,y_lines_without_mother,x_start_without_mother,x_end_without_mother,there_is_sep_with_child,y_lines_with_child_without_mother,x_start_with_child_without_mother,x_end_with_child_without_mother=return_x_start_end_mothers_childs_and_type_of_reading_order(x_min_hor_some,x_max_hor_some,cy_hor_some,peaks_neg_tot,cy_hor_diff)
|
reading_order_type,x_starting,x_ending,y_type_2,y_diff_type_2,y_lines_without_mother,x_start_without_mother,x_end_without_mother,there_is_sep_with_child,y_lines_with_child_without_mother,x_start_with_child_without_mother,x_end_with_child_without_mother=return_x_start_end_mothers_childs_and_type_of_reading_order(x_min_hor_some,x_max_hor_some,cy_hor_some,peaks_neg_tot,cy_hor_diff)
|
||||||
|
|
||||||
|
|
||||||
|
@ -2237,4 +2239,4 @@ def return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_witho
|
||||||
#else:
|
#else:
|
||||||
#boxes.append([ 0, regions_without_separators[:,:].shape[1] ,splitter_y_new[i],splitter_y_new[i+1]])
|
#boxes.append([ 0, regions_without_separators[:,:].shape[1] ,splitter_y_new[i],splitter_y_new[i+1]])
|
||||||
|
|
||||||
return boxes
|
return boxes, peaks_neg_tot_tables
|
||||||
|
|
|
@ -109,7 +109,21 @@ def find_new_features_of_contours(contours_main):
|
||||||
# dis_x=np.abs(x_max_main-x_min_main)
|
# dis_x=np.abs(x_max_main-x_min_main)
|
||||||
|
|
||||||
return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin
|
return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin
|
||||||
|
def find_features_of_contours(contours_main):
|
||||||
|
|
||||||
|
|
||||||
|
areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))])
|
||||||
|
M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))]
|
||||||
|
cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))]
|
||||||
|
cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))]
|
||||||
|
x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))])
|
||||||
|
x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))])
|
||||||
|
|
||||||
|
y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))])
|
||||||
|
y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))])
|
||||||
|
|
||||||
|
|
||||||
|
return y_min_main, y_max_main
|
||||||
def return_parent_contours(contours, hierarchy):
|
def return_parent_contours(contours, hierarchy):
|
||||||
contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1]
|
contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1]
|
||||||
return contours_parent
|
return contours_parent
|
||||||
|
|
|
@ -52,20 +52,21 @@ def rotate_image_different( img, slope):
|
||||||
img_rotation = cv2.warpAffine(img, rotation_matrix, (num_cols, num_rows))
|
img_rotation = cv2.warpAffine(img, rotation_matrix, (num_cols, num_rows))
|
||||||
return img_rotation
|
return img_rotation
|
||||||
|
|
||||||
def rotate_max_area(image, rotated, rotated_textline, rotated_layout, angle):
|
def rotate_max_area(image, rotated, rotated_textline, rotated_layout, rotated_table_prediction, angle):
|
||||||
wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle))
|
wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle))
|
||||||
h, w, _ = rotated.shape
|
h, w, _ = rotated.shape
|
||||||
y1 = h // 2 - int(hr / 2)
|
y1 = h // 2 - int(hr / 2)
|
||||||
y2 = y1 + int(hr)
|
y2 = y1 + int(hr)
|
||||||
x1 = w // 2 - int(wr / 2)
|
x1 = w // 2 - int(wr / 2)
|
||||||
x2 = x1 + int(wr)
|
x2 = x1 + int(wr)
|
||||||
return rotated[y1:y2, x1:x2], rotated_textline[y1:y2, x1:x2], rotated_layout[y1:y2, x1:x2]
|
return rotated[y1:y2, x1:x2], rotated_textline[y1:y2, x1:x2], rotated_layout[y1:y2, x1:x2], rotated_table_prediction[y1:y2, x1:x2]
|
||||||
|
|
||||||
def rotation_not_90_func(img, textline, text_regions_p_1, thetha):
|
def rotation_not_90_func(img, textline, text_regions_p_1, table_prediction, thetha):
|
||||||
rotated = imutils.rotate(img, thetha)
|
rotated = imutils.rotate(img, thetha)
|
||||||
rotated_textline = imutils.rotate(textline, thetha)
|
rotated_textline = imutils.rotate(textline, thetha)
|
||||||
rotated_layout = imutils.rotate(text_regions_p_1, thetha)
|
rotated_layout = imutils.rotate(text_regions_p_1, thetha)
|
||||||
return rotate_max_area(img, rotated, rotated_textline, rotated_layout, thetha)
|
rotated_table_prediction = imutils.rotate(table_prediction, thetha)
|
||||||
|
return rotate_max_area(img, rotated, rotated_textline, rotated_layout, rotated_table_prediction, thetha)
|
||||||
|
|
||||||
def rotation_not_90_func_full_layout(img, textline, text_regions_p_1, text_regions_p_fully, thetha):
|
def rotation_not_90_func_full_layout(img, textline, text_regions_p_1, text_regions_p_fully, thetha):
|
||||||
rotated = imutils.rotate(img, thetha)
|
rotated = imutils.rotate(img, thetha)
|
||||||
|
|
|
@ -141,7 +141,7 @@ class EynollahXmlWriter():
|
||||||
with open(out_fname, 'w') as f:
|
with open(out_fname, 'w') as f:
|
||||||
f.write(to_xml(pcgts))
|
f.write(to_xml(pcgts))
|
||||||
|
|
||||||
def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_texline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml):
|
def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_texline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables):
|
||||||
self.logger.debug('enter build_pagexml_no_full_layout')
|
self.logger.debug('enter build_pagexml_no_full_layout')
|
||||||
|
|
||||||
# create the file structure
|
# create the file structure
|
||||||
|
@ -189,6 +189,16 @@ class EynollahXmlWriter():
|
||||||
points_co += str(int((polygons_lines_to_be_written_in_xml[mm][lmm,0,1] ) / self.scale_y))
|
points_co += str(int((polygons_lines_to_be_written_in_xml[mm][lmm,0,1] ) / self.scale_y))
|
||||||
points_co += ' '
|
points_co += ' '
|
||||||
sep_hor.get_Coords().set_points(points_co[:-1])
|
sep_hor.get_Coords().set_points(points_co[:-1])
|
||||||
|
for mm in range(len(found_polygons_tables)):
|
||||||
|
tab_region = TableRegionType(id=counter.next_region_id, Coords=CoordsType())
|
||||||
|
page.add_TableRegion(tab_region)
|
||||||
|
points_co = ''
|
||||||
|
for lmm in range(len(found_polygons_tables[mm])):
|
||||||
|
points_co += str(int((found_polygons_tables[mm][lmm,0,0] ) / self.scale_x))
|
||||||
|
points_co += ','
|
||||||
|
points_co += str(int((found_polygons_tables[mm][lmm,0,1] ) / self.scale_y))
|
||||||
|
points_co += ' '
|
||||||
|
tab_region.get_Coords().set_points(points_co[:-1])
|
||||||
|
|
||||||
return pcgts
|
return pcgts
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue