qurator-spk/eynollah
1
0
Fork 0
mirror of https://github.com/qurator-spk/eynollah.git synced 2026-03-24 08:02:45 +01:00
Code Issues Projects Releases Wiki Activity

drop ProcessPoolExecutor for intra-page parallel subprocessing…

Browse source 
(interferes with inter-page parallelism, not as useful)
This commit is contained in:
Robert Sachunsky 2026-03-13 02:38:40 +01:00
parent becf031c65
commit ae0f194241
2 changed files with 25 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/eynollah/eynollah.py
View file

@ -26,10 +26,8 @@ import time
from typing import Optional from typing import Optional
from functools import partial from functools import partial
from pathlib import Path from pathlib import Path
from multiprocessing import cpu_count
import gc import gc
from concurrent.futures import ProcessPoolExecutor
import cv2 import cv2
import numpy as np import numpy as np
import shapely.affinity import shapely.affinity
@ -147,9 +145,6 @@ class Eynollah:
else: else:
self.num_col_lower = num_col_lower self.num_col_lower = num_col_lower
# for parallelization of CPU-intensive tasks:
self.executor = ProcessPoolExecutor(max_workers=cpu_count())
if threshold_art_class_layout: if threshold_art_class_layout:
self.threshold_art_class_layout = float(threshold_art_class_layout) self.threshold_art_class_layout = float(threshold_art_class_layout)
else: else:
@ -195,9 +190,6 @@ class Eynollah:
self.model_zoo.get(model).output_shape) self.model_zoo.get(model).output_shape)
def __del__(self): def __del__(self):
if executor := getattr(self, 'executor', None):
executor.shutdown()
del self.executor
if model_zoo := getattr(self, 'model_zoo', None): if model_zoo := getattr(self, 'model_zoo', None):
if shutdown := getattr(model_zoo, 'shutdown', None): if shutdown := getattr(model_zoo, 'shutdown', None):
shutdown() shutdown()
@ -1039,22 +1031,19 @@ class Eynollah:
if not len(contours_par): if not len(contours_par):
return [], [], [] return [], [], []
self.logger.debug("enter get_slopes_and_deskew_new_curved") self.logger.debug("enter get_slopes_and_deskew_new_curved")
with share_ndarray(textline_mask_tot) as textline_mask_tot_shared: results = map(partial(do_work_of_slopes_new_curved,
with share_ndarray(mask_texts_only) as mask_texts_only_shared: textline_mask_tot_ea=textline_mask_tot,
assert self.executor mask_texts_only=mask_texts_only,
results = self.executor.map(partial(do_work_of_slopes_new_curved, num_col=num_col,
textline_mask_tot_ea=textline_mask_tot_shared, scale_par=scale_par,
mask_texts_only=mask_texts_only_shared, slope_deskew=slope_deskew,
num_col=num_col, MAX_SLOPE=MAX_SLOPE,
scale_par=scale_par, KERNEL=KERNEL,
slope_deskew=slope_deskew, logger=self.logger,
MAX_SLOPE=MAX_SLOPE, plotter=self.plotter,
KERNEL=KERNEL, name=name),
logger=self.logger, boxes, contours_par)
plotter=self.plotter, results = list(results) # exhaust prior to release
name=name),
boxes, contours_par)
results = list(results) # exhaust prior to release
#textline_polygons, box_coord, slopes = zip(*results) #textline_polygons, box_coord, slopes = zip(*results)
self.logger.debug("exit get_slopes_and_deskew_new_curved") self.logger.debug("exit get_slopes_and_deskew_new_curved")
return tuple(zip(*results)) return tuple(zip(*results))
@ -1628,7 +1617,7 @@ class Eynollah:
def run_deskew(self, textline_mask_tot_ea): def run_deskew(self, textline_mask_tot_ea):
#print(textline_mask_tot_ea.shape, 'textline_mask_tot_ea deskew') #print(textline_mask_tot_ea.shape, 'textline_mask_tot_ea deskew')
slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True, slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True,
map=self.executor.map, logger=self.logger, plotter=self.plotter) logger=self.logger, plotter=self.plotter)
self.logger.info("slope_deskew: %.2f°", slope_deskew) self.logger.info("slope_deskew: %.2f°", slope_deskew)
return slope_deskew return slope_deskew

32
src/eynollah/utils/separate_lines.py
View file

@ -15,7 +15,6 @@ from .contour import (
return_contours_of_interested_textline, return_contours_of_interested_textline,
find_contours_mean_y_diff, find_contours_mean_y_diff,
) )
from .shm import share_ndarray, wrap_ndarray_shared
from . import ( from . import (
find_num_col_deskew, find_num_col_deskew,
box2rect, box2rect,
@ -1493,7 +1492,6 @@ def separate_lines_new2(img_crop, thetha, num_col, slope_region, logger=None, pl
return img_patch_interest_revised return img_patch_interest_revised
@wrap_ndarray_shared(kw='img')
def do_image_rotation(angle, img=None, sigma_des=1.0, logger=None): def do_image_rotation(angle, img=None, sigma_des=1.0, logger=None):
if logger is None: if logger is None:
logger = getLogger(__package__) logger = getLogger(__package__)
@ -1507,7 +1505,7 @@ def do_image_rotation(angle, img=None, sigma_des=1.0, logger=None):
return var return var
def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100,
main_page=False, logger=None, plotter=None, name=None, map=None): main_page=False, logger=None, plotter=None, name=None):
if main_page and plotter: if main_page and plotter:
plotter.save_plot_of_textline_density(img_patch_org, name) plotter.save_plot_of_textline_density(img_patch_org, name)
@ -1527,16 +1525,16 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100,
if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]: if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]:
angles = np.array([-45, 0, 45, 90,]) angles = np.array([-45, 0, 45, 90,])
angle, _ = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, name=name, plotter=plotter) angle, _ = get_smallest_skew(img_resized, sigma_des, angles, logger=logger, name=name, plotter=plotter)
angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles) angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles)
angle, _ = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, name=name, plotter=plotter) angle, _ = get_smallest_skew(img_resized, sigma_des, angles, logger=logger, name=name, plotter=plotter)
elif main_page: elif main_page:
#angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) #angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45])
angles = np.concatenate((np.linspace(-12, -7, n_tot_angles // 4), angles = np.concatenate((np.linspace(-12, -7, n_tot_angles // 4),
np.linspace(-6, 6, n_tot_angles // 2), np.linspace(-6, 6, n_tot_angles // 2),
np.linspace(7, 12, n_tot_angles // 4))) np.linspace(7, 12, n_tot_angles // 4)))
angle, var = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, name=name, plotter=plotter) angle, var = get_smallest_skew(img_resized, sigma_des, angles, logger=logger, name=name, plotter=plotter)
early_slope_edge=11 early_slope_edge=11
if abs(angle) > early_slope_edge: if abs(angle) > early_slope_edge:
@ -1544,12 +1542,12 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100,
angles2 = np.linspace(-90, -12, n_tot_angles) angles2 = np.linspace(-90, -12, n_tot_angles)
else: else:
angles2 = np.linspace(90, 12, n_tot_angles) angles2 = np.linspace(90, 12, n_tot_angles)
angle2, var2 = get_smallest_skew(img_resized, sigma_des, angles2, map=map, logger=logger, name=name, plotter=plotter) angle2, var2 = get_smallest_skew(img_resized, sigma_des, angles2, logger=logger, name=name, plotter=plotter)
if var2 > var: if var2 > var:
angle = angle2 angle = angle2
else: else:
angles = np.linspace(-25, 25, int(0.5 * n_tot_angles) + 10) angles = np.linspace(-25, 25, int(0.5 * n_tot_angles) + 10)
angle, var = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, name=name, plotter=plotter) angle, var = get_smallest_skew(img_resized, sigma_des, angles, logger=logger, name=name, plotter=plotter)
early_slope_edge=22 early_slope_edge=22
if abs(angle) > early_slope_edge: if abs(angle) > early_slope_edge:
@ -1557,24 +1555,19 @@ def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100,
angles2 = np.linspace(-90, -25, int(0.5 * n_tot_angles) + 10) angles2 = np.linspace(-90, -25, int(0.5 * n_tot_angles) + 10)
else: else:
angles2 = np.linspace(90, 25, int(0.5 * n_tot_angles) + 10) angles2 = np.linspace(90, 25, int(0.5 * n_tot_angles) + 10)
angle2, var2 = get_smallest_skew(img_resized, sigma_des, angles2, map=map, logger=logger, name=name, plotter=plotter) angle2, var2 = get_smallest_skew(img_resized, sigma_des, angles2, logger=logger, name=name, plotter=plotter)
if var2 > var: if var2 > var:
angle = angle2 angle = angle2
# precision stage: # precision stage:
angles = np.linspace(angle - 2.5, angle + 2.5, n_tot_angles // 2) angles = np.linspace(angle - 2.5, angle + 2.5, n_tot_angles // 2)
angle, _ = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, name=name, plotter=plotter) angle, _ = get_smallest_skew(img_resized, sigma_des, angles, logger=logger, name=name, plotter=plotter)
return angle return angle
def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, name=None, map=map): def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, name=None):
if logger is None: if logger is None:
logger = getLogger(__package__) logger = getLogger(__package__)
if map is None: results = [do_image_rotation(angle, img=img, sigma_des=sigma_des, logger=logger)
results = [do_image_rotation.__wrapped__(angle, img=img, sigma_des=sigma_des, logger=logger) for angle in angles]
for angle in angles]
else:
with share_ndarray(img) as img_shared:
results = list(map(partial(do_image_rotation, img=img_shared, sigma_des=sigma_des, logger=None),
angles))
if plotter: if plotter:
plotter.save_plot_of_rotation_angle(angles, results, name) plotter.save_plot_of_rotation_angle(angles, results, name)
try: try:
@ -1589,8 +1582,6 @@ def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, name=No
var = 0 var = 0
return angle, var return angle, var
@wrap_ndarray_shared(kw='textline_mask_tot_ea')
@wrap_ndarray_shared(kw='mask_texts_only')
def do_work_of_slopes_new_curved( def do_work_of_slopes_new_curved(
box_text, contour_par, box_text, contour_par,
textline_mask_tot_ea=None, mask_texts_only=None, textline_mask_tot_ea=None, mask_texts_only=None,
@ -1685,7 +1676,6 @@ def do_work_of_slopes_new_curved(
return textlines_cnt_per_region[::-1], crop_coor, slope return textlines_cnt_per_region[::-1], crop_coor, slope
@wrap_ndarray_shared(kw='textline_mask_tot_ea')
def do_work_of_slopes_new_light( def do_work_of_slopes_new_light(
box_text, contour, contour_par, box_text, contour, contour_par,
textline_mask_tot_ea=None, slope_deskew=0, textline_mask_tot_ea=None, slope_deskew=0,