scaling contours without dilation

qurator/eynollah/eynollah.py

@@ -256,7 +256,7 @@ class Eynollah:
         ##self.model_region_dir_fully_new = dir_models + "/model_2_full_layout_new_trans"
         self.model_region_dir_fully = dir_models + "/modelens_full_layout_24_till_28"#"/model_2_full_layout_new_trans"
         if self.textline_light:
-            self.model_textline_dir = dir_models + "/model_textline_ens_3_4_5_6_artificial"#"/modelens_textline_9_12_13_14_15"#"/eynollah-textline_light_20210425"#
+            self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024"#"/modelens_textline_1_4_16092024"#"/model_textline_ens_3_4_5_6_artificial"#"/modelens_textline_1_3_4_20240915"#"/model_textline_ens_3_4_5_6_artificial"#"/modelens_textline_9_12_13_14_15"#"/eynollah-textline_light_20210425"#
         else:
             self.model_textline_dir = dir_models + "/eynollah-textline_20210425"
         if self.ocr:
@@ -796,7 +796,7 @@ class Eynollah:
 
         return model, None
 
-    def do_prediction(self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False):
+    def do_prediction(self, patches, img, model, n_batch_inference=1, marginal_of_patch_percent=0.1, thresholding_for_some_classes_in_light_version=False, thresholding_for_artificial_class_in_light_version=False):
         self.logger.debug("enter do_prediction")
 
         img_height_model = model.layers[len(model.layers) - 1].output_shape[1]
@@ -903,6 +903,13 @@ class Eynollah:
                             seg[seg_not_base==1]=4
                             seg[seg_background==1]=0
                             seg[(seg_line==1) & (seg==0)]=3
+                        if thresholding_for_artificial_class_in_light_version:
+                            seg_art = label_p_pred[:,:,:,2]
+                            
+                            seg_art[seg_art<0.2] = 0
+                            seg_art[seg_art>0] =1
+                            
+                            seg[seg_art==1]=2
                         
                         indexer_inside_batch = 0
                         for i_batch, j_batch in zip(list_i_s, list_j_s):
@@ -978,6 +985,14 @@ class Eynollah:
                             seg[seg_background==1]=0
                             seg[(seg_line==1) & (seg==0)]=3
                             
+                        if thresholding_for_artificial_class_in_light_version:
+                            seg_art = label_p_pred[:,:,:,2]
+                            
+                            seg_art[seg_art<0.2] = 0
+                            seg_art[seg_art>0] =1
+                            
+                            seg[seg_art==1]=2
+                        
                         indexer_inside_batch = 0
                         for i_batch, j_batch in zip(list_i_s, list_j_s):
                             seg_in = seg[indexer_inside_batch,:,:]
@@ -1845,30 +1860,35 @@ class Eynollah:
 
     def textline_contours(self, img, patches, scaler_h, scaler_w, num_col_classifier=None):
         self.logger.debug('enter textline_contours')
+        thresholding_for_artificial_class_in_light_version = True#False
         if not self.dir_in:
             model_textline, session_textline = self.start_new_session_and_model(self.model_textline_dir if patches else self.model_textline_dir_np)
-        img = img.astype(np.uint8)
+        #img = img.astype(np.uint8)
         img_org = np.copy(img)
         img_h = img_org.shape[0]
         img_w = img_org.shape[1]
         img = resize_image(img_org, int(img_org.shape[0] * scaler_h), int(img_org.shape[1] * scaler_w))
-        #print(img.shape,'bin shape textline')
+        
         if not self.dir_in:
-            prediction_textline = self.do_prediction(patches, img, model_textline, marginal_of_patch_percent=0.2, n_batch_inference=3)
+            prediction_textline = self.do_prediction(patches, img, model_textline, marginal_of_patch_percent=0.2, n_batch_inference=3, thresholding_for_artificial_class_in_light_version=thresholding_for_artificial_class_in_light_version)
-            if num_col_classifier==1:
+            
-                prediction_textline_nopatch = self.do_prediction(False, img, model_textline)
+            #if not thresholding_for_artificial_class_in_light_version:
-                prediction_textline[:,:][prediction_textline_nopatch[:,:]==0] = 0
+                #if num_col_classifier==1:
-        else:
+                    #prediction_textline_nopatch = self.do_prediction(False, img, model_textline)
-            prediction_textline = self.do_prediction(patches, img, self.model_textline, marginal_of_patch_percent=0.2, n_batch_inference=3)
+                    #prediction_textline[:,:][prediction_textline_nopatch[:,:]==0] = 0
-            if num_col_classifier==1:
+        else:
-                prediction_textline_nopatch = self.do_prediction(False, img, model_textline)
+            prediction_textline = self.do_prediction(patches, img, self.model_textline, marginal_of_patch_percent=0.2, n_batch_inference=3,thresholding_for_artificial_class_in_light_version=thresholding_for_artificial_class_in_light_version)
-                prediction_textline[:,:][prediction_textline_nopatch[:,:]==0] = 0
+            #if not thresholding_for_artificial_class_in_light_version:
+                #if num_col_classifier==1:
+                    #prediction_textline_nopatch = self.do_prediction(False, img, model_textline)
+                    #prediction_textline[:,:][prediction_textline_nopatch[:,:]==0] = 0
         prediction_textline = resize_image(prediction_textline, img_h, img_w)
         
         textline_mask_tot_ea_art = (prediction_textline[:,:]==2)*1
         
         old_art = np.copy(textline_mask_tot_ea_art)
         
+        if not thresholding_for_artificial_class_in_light_version:
             textline_mask_tot_ea_art = textline_mask_tot_ea_art.astype('uint8')
             textline_mask_tot_ea_art = cv2.dilate(textline_mask_tot_ea_art, KERNEL, iterations=1)
             
@@ -1876,10 +1896,13 @@ class Eynollah:
         
         textline_mask_tot_ea_lines = (prediction_textline[:,:]==1)*1
         textline_mask_tot_ea_lines = textline_mask_tot_ea_lines.astype('uint8')
+        
+        if not thresholding_for_artificial_class_in_light_version:
             textline_mask_tot_ea_lines = cv2.dilate(textline_mask_tot_ea_lines, KERNEL, iterations=1)
         
         prediction_textline[:,:][textline_mask_tot_ea_lines[:,:]==1]=1
         
+        if not thresholding_for_artificial_class_in_light_version:
             prediction_textline[:,:][old_art[:,:]==1]=2
         
         if not self.dir_in:
@@ -1959,7 +1982,7 @@ class Eynollah:
             img_w_new = 2300#3000
             img_h_new = int(img_org.shape[0] / float(img_org.shape[1]) * img_w_new)
         else:
-            img_w_new = 3300#4000
+            img_w_new = 3000#4000
             img_h_new = int(img_org.shape[0] / float(img_org.shape[1]) * img_w_new)
         img_resized = resize_image(img,img_h_new, img_w_new )
         
@@ -1968,7 +1991,7 @@ class Eynollah:
         #if (not self.input_binary) or self.full_layout:
         #if self.input_binary:
             #img_bin = np.copy(img_resized)
-        if (not self.input_binary and self.full_layout) or (not self.input_binary and num_col_classifier >= 3):
+        if (not self.input_binary and self.full_layout):# or (not self.input_binary and num_col_classifier >= 3):
             if not self.dir_in:
                 model_bin, session_bin = self.start_new_session_and_model(self.model_dir_of_binarization)
                 prediction_bin = self.do_prediction(True, img_resized, model_bin, n_batch_inference=5)
@@ -3795,6 +3818,137 @@ class Eynollah:
     def return_list_of_contours_with_desired_order(self, ls_cons, sorted_indexes):
         return [ls_cons[sorted_indexes[index]] for index in range(len(sorted_indexes))]
     
+    def scale_contours(self,all_found_textline_polygons):
+        for i in range(len(all_found_textline_polygons[0])):
+            con_ind = all_found_textline_polygons[0][i]
+            x_min = np.min( con_ind[:,0,0] )
+            y_min = np.min( con_ind[:,0,1] )
+            
+            x_max = np.max( con_ind[:,0,0] )
+            y_max = np.max( con_ind[:,0,1] )
+            
+            x_mean = np.mean( con_ind[:,0,0] )
+            y_mean = np.mean( con_ind[:,0,1] )
+            
+            arg_y_max = np.argmax( con_ind[:,0,1] )
+            arg_y_min = np.argmin( con_ind[:,0,1] )
+            
+            x_cor_y_max = con_ind[arg_y_max,0,0]
+            x_cor_y_min = con_ind[arg_y_min,0,0]
+            
+            m_con = (y_max - y_min) / float(x_cor_y_max - x_cor_y_min)
+            
+            con_scaled = con_ind*1
+            
+            con_scaled = con_scaled.astype(np.float)
+            
+            con_scaled[:,0,0] = con_scaled[:,0,0] - int(x_mean)
+            con_scaled[:,0,1] = con_scaled[:,0,1] - int(y_mean)
+            
+
+            if (x_max - x_min) > (y_max - y_min):
+            
+                if (y_max-y_min)<=15:
+                    con_scaled[:,0,1] = con_ind[:,0,1]*1.8
+                    
+                    y_max_scaled = np.max(con_scaled[:,0,1])
+                    y_min_scaled = np.min(con_scaled[:,0,1])
+                    
+                    y_max_expected = ( m_con*1.8*(x_cor_y_max-x_cor_y_min) + y_min_scaled )
+                elif (y_max-y_min)<=30 and (y_max-y_min)>15:
+                    con_scaled[:,0,1] = con_ind[:,0,1]*1.6
+                    y_max_scaled = np.max(con_scaled[:,0,1])
+                    y_min_scaled = np.min(con_scaled[:,0,1])
+                    
+                    y_max_expected = ( m_con*1.6*(x_cor_y_max-x_cor_y_min) + y_min_scaled )
+                elif (y_max-y_min)>30 and (y_max-y_min)<100:
+                    con_scaled[:,0,1] = con_ind[:,0,1]*1.35
+                    y_max_scaled = np.max(con_scaled[:,0,1])
+                    y_min_scaled = np.min(con_scaled[:,0,1])
+                    
+                    y_max_expected = ( m_con*1.35*(x_cor_y_max-x_cor_y_min) + y_min_scaled )
+                else:
+                    con_scaled[:,0,1] = con_ind[:,0,1]*1.2
+                    y_max_scaled = np.max(con_scaled[:,0,1])
+                    y_min_scaled = np.min(con_scaled[:,0,1])
+                    
+                    y_max_expected = ( m_con*1.2*(x_cor_y_max-x_cor_y_min) + y_min_scaled )
+                con_scaled[:,0,0] = con_ind[:,0,0]*1.03
+                
+
+                
+                
+                if y_max_expected<=y_max_scaled:
+                    con_scaled[:,0,1] = con_scaled[:,0,1] - y_min_scaled
+                    
+                    con_scaled[:,0,1] = con_scaled[:,0,1]*(y_max_expected - y_min_scaled)/ (y_max_scaled - y_min_scaled)
+                    con_scaled[:,0,1] = con_scaled[:,0,1] + y_min_scaled
+                
+            else:
+                
+                if (x_max-x_min)<=15:
+                    con_scaled[:,0,0] = con_ind[:,0,0]*1.8
+                elif (x_max-x_min)<=30 and (x_max-x_min)>15:
+                    con_scaled[:,0,0] = con_ind[:,0,0]*1.6
+                elif (x_max-x_min)>30 and (x_max-x_min)<100:
+                    con_scaled[:,0,0] = con_ind[:,0,0]*1.35
+                else:
+                    con_scaled[:,0,0] = con_ind[:,0,0]*1.2
+                con_scaled[:,0,1] = con_ind[:,0,1]*1.03
+                
+        
+            x_min_n = np.min( con_scaled[:,0,0] )
+            y_min_n = np.min( con_scaled[:,0,1] )
+            
+            x_mean_n = np.mean( con_scaled[:,0,0] )
+            y_mean_n = np.mean( con_scaled[:,0,1] )
+        
+            ##diff_x = (x_min_n - x_min)*1
+            ##diff_y = (y_min_n - y_min)*1
+            
+            diff_x = (x_mean_n - x_mean)*1
+            diff_y = (y_mean_n - y_mean)*1
+            
+            
+            con_scaled[:,0,0] = (con_scaled[:,0,0] - diff_x) 
+            con_scaled[:,0,1] = (con_scaled[:,0,1] - diff_y)
+            
+            x_max_n = np.max( con_scaled[:,0,0] )
+            y_max_n = np.max( con_scaled[:,0,1] )
+            
+            diff_disp_x  = (x_max_n - x_max) / 2.
+            diff_disp_y  = (y_max_n - y_max) / 2.
+            
+            x_vals = np.array( np.abs(con_scaled[:,0,0] - diff_disp_x) ).astype(np.int16)
+            y_vals = np.array( np.abs(con_scaled[:,0,1] - diff_disp_y) ).astype(np.int16)
+            all_found_textline_polygons[0][i][:,0,0] = x_vals[:]
+            all_found_textline_polygons[0][i][:,0,1] = y_vals[:]
+        return all_found_textline_polygons
+    
+    def scale_contours_new(self, textline_mask_tot_ea):
+        
+        cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(textline_mask_tot_ea)
+        all_found_textline_polygons1 = filter_contours_area_of_image(textline_mask_tot_ea, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001)
+    
+        
+        textline_mask_tot_ea_res = resize_image(textline_mask_tot_ea, int( textline_mask_tot_ea.shape[0]*1.6),  textline_mask_tot_ea.shape[1])
+        cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(textline_mask_tot_ea_res)
+        ##all_found_textline_polygons = filter_contours_area_of_image(textline_mask_tot_ea_res, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001)
+        all_found_textline_polygons = filter_contours_area_of_image(textline_mask_tot_ea_res, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001)
+        
+        for i in range(len(all_found_textline_polygons)):
+            
+            #x_mean_1 = np.mean( all_found_textline_polygons1[i][:,0,0] )
+            y_mean_1 = np.mean( all_found_textline_polygons1[i][:,0,1] )
+            
+            #x_mean = np.mean( all_found_textline_polygons[i][:,0,0] )
+            y_mean = np.mean( all_found_textline_polygons[i][:,0,1] )
+            
+            ydiff = y_mean - y_mean_1
+            
+            all_found_textline_polygons[i][:,0,1] = all_found_textline_polygons[i][:,0,1] - ydiff
+        return all_found_textline_polygons
+    
     
     def run(self):
         """
@@ -3802,7 +3956,7 @@ class Eynollah:
         """
         self.logger.debug("enter run")
         
-        skip_layout_ro = False#True
+        skip_layout_ro = True
 
         t0_tot = time.time()
 
@@ -3820,7 +3974,6 @@ class Eynollah:
             self.logger.info("Enhancing took %.1fs ", time.time() - t0)
             #print("text region early -1 in %.1fs", time.time() - t0)
             t1 = time.time()
-            
             if not skip_layout_ro:
                 if self.light_version:
                     text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light = self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier)
@@ -4032,6 +4185,7 @@ class Eynollah:
                         if self.textline_light:
                             slopes, all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, index_by_text_par_con = self.get_slopes_and_deskew_new_light(txt_con_org, contours_only_text_parent, textline_mask_tot_ea_org, image_page_rotated, boxes_text, slope_deskew)
                             slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, all_box_coord_marginals, _ = self.get_slopes_and_deskew_new_light(polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_org, image_page_rotated, boxes_marginals, slope_deskew)
+                            
                         else:
                             slopes, all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, index_by_text_par_con = self.get_slopes_and_deskew_new_light(txt_con_org, contours_only_text_parent, textline_mask_tot_ea, image_page_rotated, boxes_text, slope_deskew)
                             slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, all_box_coord_marginals, _ = self.get_slopes_and_deskew_new_light(polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, image_page_rotated, boxes_marginals, slope_deskew)
@@ -4212,10 +4366,17 @@ class Eynollah:
                 
                 page_coord, image_page, textline_mask_tot_ea, img_bin_light, cont_page = self.run_graphics_and_columns_without_layout(textline_mask_tot_ea, img_bin_light)
                 
+                
+                ##all_found_textline_polygons =self.scale_contours_new(textline_mask_tot_ea)
+                
                 cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(textline_mask_tot_ea)
                 all_found_textline_polygons = filter_contours_area_of_image(textline_mask_tot_ea, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001)
                 
                 all_found_textline_polygons=[ all_found_textline_polygons ]
+                
+                all_found_textline_polygons = self.scale_contours(all_found_textline_polygons)
+                
+                
                 order_text_new = [0]
                 slopes =[0]
                 id_of_texts_tot =['region_0001']