switch from stdlib to loky.ProcessPoolExecutor, ensure shutdown

pull/142/head
Robert Sachunsky 4 months ago
parent f93c6c288d
commit 0ae28f7d3e

@ -5,3 +5,4 @@ scikit-learn >= 0.23.2
tensorflow < 2.13 tensorflow < 2.13
imutils >= 0.5.3 imutils >= 0.5.3
numba <= 0.58.1 numba <= 0.58.1
loky

@ -10,11 +10,12 @@ import math
import os import os
import sys import sys
import time import time
import atexit
import warnings import warnings
from functools import partial from functools import partial
from pathlib import Path from pathlib import Path
from multiprocessing import cpu_count from multiprocessing import cpu_count
from concurrent.futures import ProcessPoolExecutor from loky import ProcessPoolExecutor
import gc import gc
from ocrd_utils import getLogger from ocrd_utils import getLogger
import cv2 import cv2
@ -257,7 +258,8 @@ class Eynollah:
pcgts=pcgts) pcgts=pcgts)
self.logger = logger if logger else getLogger('eynollah') self.logger = logger if logger else getLogger('eynollah')
# for parallelization of CPU-intensive tasks: # for parallelization of CPU-intensive tasks:
self.executor = ProcessPoolExecutor(max_workers=cpu_count()) self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200)
atexit.register(self.executor.shutdown)
self.dir_models = dir_models self.dir_models = dir_models
self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425"
self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425"

Loading…
Cancel
Save