diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..72b2c5a --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,28 @@ +version: 2 + +jobs: + + build-python36: + docker: + - image: python:3.6 + steps: + - checkout + - restore_cache: + keys: + - model-cache + - run: make models + - save_cache: + key: model-cache + paths: + models_eynollah.tar.gz + models_eynollah + - run: make install + - run: make smoke-test + +workflows: + version: 2 + build: + jobs: + - build-python36 + #- build-python37 + #- build-python38 # no tensorflow for python 3.8 diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 562fb6f..0000000 --- a/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -tests -dist -build -env* -*.egg-info -models_eynollah* diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml deleted file mode 100644 index d77958b..0000000 --- a/.github/workflows/build-docker.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: CD - -on: - push: - branches: [ "master" ] - workflow_dispatch: # run manually - -jobs: - - build: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - # we need tags for docker version tagging - fetch-tags: true - fetch-depth: 0 - - # Activate cache export feature to reduce build time of images - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERIO_USERNAME }} - password: ${{ secrets.DOCKERIO_PASSWORD }} - - name: Build the Docker image - # build both tags at the same time - run: make docker DOCKER_TAG="docker.io/ocrd/eynollah -t ghcr.io/qurator-spk/eynollah" - - name: Test the Docker image - run: docker run --rm ocrd/eynollah ocrd-eynollah-segment -h - - name: Push to Dockerhub - run: docker push docker.io/ocrd/eynollah - - name: Push to Github Container Registry - run: docker push ghcr.io/qurator-spk/eynollah diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml deleted file mode 100644 index 248f4ef..0000000 --- a/.github/workflows/pypi.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: PyPI CD - -on: - release: - types: [published] - workflow_dispatch: - -jobs: - pypi-publish: - name: upload release to PyPI - runs-on: ubuntu-latest - permissions: - # IMPORTANT: this permission is mandatory for Trusted Publishing - id-token: write - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - - name: Build package - run: make build - - name: Publish package distributions to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true diff --git a/.github/workflows/test-eynollah.yml b/.github/workflows/test-eynollah.yml index b27586c..1afd2a6 100644 --- a/.github/workflows/test-eynollah.yml +++ b/.github/workflows/test-eynollah.yml @@ -1,9 +1,9 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Test +name: Python package -on: [push] +on: [push, pull_request] jobs: build: @@ -11,64 +11,26 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.6'] # '3.7' steps: - - name: clean up - run: | - df -h - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - df -h - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - id: seg_model_cache + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + id: model_cache with: path: models_eynollah key: ${{ runner.os }}-models - - uses: actions/cache@v4 - id: bin_model_cache - with: - path: default-2021-03-09 - key: ${{ runner.os }}-modelbin - name: Download models - if: steps.seg_model_cache.outputs.cache-hit != 'true' || steps.bin_model_cache.outputs.cache-hit != 'true' + if: steps.model_cache.outputs.cache-hit != 'true' run: make models - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - make install-dev EXTRAS=OCR,plotting - make deps-test + pip install . + pip install -r requirements-test.txt - name: Test with pytest - run: make coverage PYTEST_ARGS="-vv --junitxml=pytest.xml" - - name: Get coverage results - run: | - coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - coverage html - coverage json - coverage xml - - name: Store coverage results - uses: actions/upload-artifact@v4 - with: - name: coverage-report_${{ matrix.python-version }} - path: | - htmlcov - pytest.xml - coverage.xml - coverage.json - - name: Upload coverage results - uses: codecov/codecov-action@v4 - with: - files: coverage.xml - fail_ci_if_error: false - - name: Test standalone CLI - run: make smoke-test - - name: Test OCR-D CLI - run: make ocrd-test + run: make test diff --git a/CHANGELOG.md b/CHANGELOG.md index ad86fe5..c6a5193 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,144 +5,7 @@ Versioned according to [Semantic Versioning](http://semver.org/). ## Unreleased -Fixed: - - * restoring the contour in the original image caused an error due to an empty tuple - -## [0.4.0] - 2025-04-07 - -Fixed: - - * allow empty imports for optional dependencies - * avoid Numpy warnings (empty slices etc) - * remove deprecated Numpy types - * binarization CLI: make `dir_in` usable again - -Added: - - * Continuous Deployment via Dockerhub and GHCR - * CI: also test CLIs and OCR-D - * CI: measure code coverage, annotate+upload reports - * smoke-test: also check results - * smoke-test: also test sbb-binarize - * ocrd-test: analog for OCR-D CLI (segment and binarize) - * pytest: add asserts, extend coverage, use subtests for various options - * pytest: also add binarization - * pytest: add `dir_in` mode (segment and binarize) - * make install: control optional dependencies via `EXTRAS` variable - * OCR-D: expose and describe recently added parameters: - - `ignore_page_extraction` - - `allow_enhancement` - - `textline_light` - - `right_to_left` - * OCR-D: :fire: integrate ocrd-sbb-binarize - * add detection confidence in `TextRegion/Coords/@conf` - (but only in light version and not for marginalia) - -Changed: - - * Docker build: simplify, w/ `OCR`, conform to OCR-D spec - * OCR-D: :fire: migrate to core v3 - - initialize+setup only once - - restrict number of parallel page workers to 1 - (conflicts with existing multiprocessing; TF parts not mp-compatible) - - do query maximally annotated page image - (but filtering existing binarization/cropping/deskewing), - rebase (as new `@imageFilename`) if necessary - - add behavioural docstring - - * :fire: refactor `Eynollah` API: - - no more data (kw)args at init, - but kwargs `dir_in` / `image_filename` for `run()` - - no more data attributes, but function kwargs - (`pcgts`, `image_filename`, `image_pil`, `dir_in`, `override_dpi`) - - remove redundant TF session/model loaders - (only load once during init) - - factor `run_single()` out of `run()` (loop body), - expose for independent calls (like OCR-D) - - expose `cache_images()`, add `dpi` kwarg, set `self._imgs` - - single-image mode writes PAGE file result - (just as directory mode does) - - * CLI: assertions (instead of print+exit) for options checks - * light mode: fine-tune ratio to better detect a region as header - -## [0.3.1] - 2024-08-27 - -Fixed: - - * regression in OCR-D processor, #106 - * Expected Ptrcv::UMat for argument 'contour', #110 - * Memory usage explosion with very narrow images (e.g. book spine), #67 - -## [0.3.0] - 2023-05-13 - -Changed: - - * Eynollah light integration, #86 - * use PEP420 style qurator namespace, #97 - * set_memory_growth to all GPU devices alike, #100 - -Fixed: - - * PAGE-XML coordinates can have self-intersections, #20 - * reading order representation (XML order vs index), #22 - * allow cropping separately, #26 - * Order of regions, #51 - * error while running inference, #75 - * Eynollah crashes while processing image, #77 - * ValueError: bad marshal data, #87 - * contour extraction: inhomogeneous shape, #92 - * Confusing model dir variables, #93 - * New release?, #96 - -## [0.2.0] - 2023-03-24 - -Changed: - - * Convert default model from HDFS to TF SavedModel, #91 - -Added: - - * parmeter `tables` to toggle table detectino, #91 - * default model described in ocrd-tool.json, #91 - -## [0.1.0] - 2023-03-22 - -Fixed: - - * Do not produce spurious `TextEquiv`, #68 - * Less spammy logging, #64, #65, #71 - -Changed: - - * Upgrade to tensorflow 2.4.0, #74 - * Improved README - * CI: test for python 3.7+, #90 - -## [0.0.11] - 2022-02-02 - -Fixed: - - * `models` parameter should have `content-type`, #61, OCR-D/core#777 - -## [0.0.10] - 2021-09-27 - -Fixed: - - * call to `uild_pagexml_no_full_layout` for empty pages, #52 - -## [0.0.9] - 2021-08-16 - -Added: - - * Table detection, #48 - -Fixed: - - * Catch exception, #47 - -## [0.0.8] - 2021-07-27 +## [0.0.7] - 2021-07-27 Fixed: @@ -187,13 +50,6 @@ Fixed: Initial release -[0.3.1]: ../../compare/v0.3.1...v0.3.0 -[0.3.0]: ../../compare/v0.3.0...v0.2.0 -[0.2.0]: ../../compare/v0.2.0...v0.1.0 -[0.1.0]: ../../compare/v0.1.0...v0.0.11 -[0.0.11]: ../../compare/v0.0.11...v0.0.10 -[0.0.10]: ../../compare/v0.0.10...v0.0.9 -[0.0.9]: ../../compare/v0.0.9...v0.0.8 [0.0.8]: ../../compare/v0.0.8...v0.0.7 [0.0.7]: ../../compare/v0.0.7...v0.0.6 [0.0.6]: ../../compare/v0.0.6...v0.0.5 diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 4ba498b..0000000 --- a/Dockerfile +++ /dev/null @@ -1,47 +0,0 @@ -ARG DOCKER_BASE_IMAGE -FROM $DOCKER_BASE_IMAGE - -ARG VCS_REF -ARG BUILD_DATE -LABEL \ - maintainer="https://ocr-d.de/en/contact" \ - org.label-schema.vcs-ref=$VCS_REF \ - org.label-schema.vcs-url="https://github.com/qurator-spk/eynollah" \ - org.label-schema.build-date=$BUILD_DATE \ - org.opencontainers.image.vendor="DFG-Funded Initiative for Optical Character Recognition Development" \ - org.opencontainers.image.title="Eynollah" \ - org.opencontainers.image.description="" \ - org.opencontainers.image.source="https://github.com/qurator-spk/eynollah" \ - org.opencontainers.image.documentation="https://github.com/qurator-spk/eynollah/blob/${VCS_REF}/README.md" \ - org.opencontainers.image.revision=$VCS_REF \ - org.opencontainers.image.created=$BUILD_DATE \ - org.opencontainers.image.base.name=ocrd/core-cuda-tf2 - -ENV DEBIAN_FRONTEND=noninteractive -# set proper locales -ENV PYTHONIOENCODING=utf8 -ENV LANG=C.UTF-8 -ENV LC_ALL=C.UTF-8 - -# avoid HOME/.local/share (hard to predict USER here) -# so let XDG_DATA_HOME coincide with fixed system location -# (can still be overridden by derived stages) -ENV XDG_DATA_HOME /usr/local/share -# avoid the need for an extra volume for persistent resource user db -# (i.e. XDG_CONFIG_HOME/ocrd/resources.yml) -ENV XDG_CONFIG_HOME /usr/local/share/ocrd-resources - -WORKDIR /build/eynollah -COPY . . -COPY ocrd-tool.json . -# prepackage ocrd-tool.json as ocrd-all-tool.json -RUN ocrd ocrd-tool ocrd-tool.json dump-tools > $(dirname $(ocrd bashlib filename))/ocrd-all-tool.json -# prepackage ocrd-all-module-dir.json -RUN ocrd ocrd-tool ocrd-tool.json dump-module-dirs > $(dirname $(ocrd bashlib filename))/ocrd-all-module-dir.json -# install everything and reduce image size -RUN make install EXTRAS=OCR && rm -rf /build/eynollah -# smoke test -RUN eynollah --help - -WORKDIR /data -VOLUME /data diff --git a/Makefile b/Makefile index 73d4d34..920f15b 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,5 @@ -PYTHON ?= python3 -PIP ?= pip3 -EXTRAS ?= - -# DOCKER_BASE_IMAGE = artefakt.dev.sbb.berlin:5000/sbb/ocrd_core:v2.68.0 -DOCKER_BASE_IMAGE ?= docker.io/ocrd/core-cuda-tf2:latest -DOCKER_TAG ?= ocrd/eynollah -DOCKER ?= docker - -#SEG_MODEL := https://qurator-data.de/eynollah/2021-04-25/models_eynollah.tar.gz -#SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah_renamed.tar.gz -SEG_MODEL := https://qurator-data.de/eynollah/2022-04-05/models_eynollah.tar.gz -#SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.0/models_eynollah.tar.gz -#SEG_MODEL := https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz - -BIN_MODEL := https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip - -PYTEST_ARGS ?= -vv +EYNOLLAH_MODELS ?= $(PWD)/models_eynollah +export EYNOLLAH_MODELS # BEGIN-EVAL makefile-parser --make-help Makefile @@ -23,105 +7,37 @@ help: @echo "" @echo " Targets" @echo "" - @echo " docker Build Docker image" - @echo " build Build Python source and binary distribution" - @echo " install Install package with pip" + @echo " models Download and extract models to $(PWD)/models_eynollah" + @echo " install Install with pip" @echo " install-dev Install editable with pip" - @echo " deps-test Install test dependencies with pip" - @echo " models Download and extract models to $(CURDIR)/models_eynollah" - @echo " smoke-test Run simple CLI check" - @echo " ocrd-test Run OCR-D CLI check" @echo " test Run unit tests" @echo "" @echo " Variables" - @echo " EXTRAS comma-separated list of features (like 'OCR,plotting') for 'install' [$(EXTRAS)]" - @echo " DOCKER_TAG Docker image tag for 'docker' [$(DOCKER_TAG)]" - @echo " PYTEST_ARGS pytest args for 'test' (Set to '-s' to see log output during test execution, '-vv' to see individual tests. [$(PYTEST_ARGS)]" - @echo " SEG_MODEL URL of 'models' archive to download for segmentation 'test' [$(SEG_MODEL)]" - @echo " BIN_MODEL URL of 'models' archive to download for binarization 'test' [$(BIN_MODEL)]" @echo "" # END-EVAL # Download and extract models to $(PWD)/models_eynollah -models: models_eynollah default-2021-03-09 +models: models_eynollah models_eynollah: models_eynollah.tar.gz - tar zxf models_eynollah.tar.gz + tar xf models_eynollah.tar.gz models_eynollah.tar.gz: - wget $(SEG_MODEL) - -default-2021-03-09: $(notdir $(BIN_MODEL)) - unzip $(notdir $(BIN_MODEL)) - mkdir $@ - mv $(basename $(notdir $(BIN_MODEL))) $@ - -$(notdir $(BIN_MODEL)): - wget $(BIN_MODEL) - -build: - $(PIP) install build - $(PYTHON) -m build . + wget 'https://qurator-data.de/eynollah/models_eynollah.tar.gz' # Install with pip install: - $(PIP) install .$(and $(EXTRAS),[$(EXTRAS)]) + pip install . # Install editable with pip install-dev: - $(PIP) install -e .$(and $(EXTRAS),[$(EXTRAS)]) + pip install -e . -deps-test: models_eynollah - $(PIP) install -r requirements-test.txt - -smoke-test: TMPDIR != mktemp -d -smoke-test: tests/resources/kant_aufklaerung_1784_0020.tif - # layout analysis: - eynollah layout -i $< -o $(TMPDIR) -m $(CURDIR)/models_eynollah - fgrep -q http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15 $(TMPDIR)/$(basename $( Document Layout Analysis with Deep Learning and Heuristics - -[![PyPI Version](https://img.shields.io/pypi/v/eynollah)](https://pypi.org/project/eynollah/) -[![GH Actions Test](https://github.com/qurator-spk/eynollah/actions/workflows/test-eynollah.yml/badge.svg)](https://github.com/qurator-spk/eynollah/actions/workflows/test-eynollah.yml) -[![GH Actions Deploy](https://github.com/qurator-spk/eynollah/actions/workflows/build-docker.yml/badge.svg)](https://github.com/qurator-spk/eynollah/actions/workflows/build-docker.yml) -[![License: ASL](https://img.shields.io/github/license/qurator-spk/eynollah)](https://opensource.org/license/apache-2-0/) -[![DOI](https://img.shields.io/badge/DOI-10.1145%2F3604951.3605513-red)](https://doi.org/10.1145/3604951.3605513) +> Document Layout Analysis ![](https://user-images.githubusercontent.com/952378/102350683-8a74db80-3fa5-11eb-8c7e-f743f7d6eae2.jpg) -## Features -* Support for up to 10 segmentation classes: - * background, [page border](https://ocr-d.de/en/gt-guidelines/trans/lyRand.html), [text region](https://ocr-d.de/en/gt-guidelines/trans/lytextregion.html#textregionen__textregion_), [text line](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextLineType.html), [header](https://ocr-d.de/en/gt-guidelines/trans/lyUeberschrift.html), [image](https://ocr-d.de/en/gt-guidelines/trans/lyBildbereiche.html), [separator](https://ocr-d.de/en/gt-guidelines/trans/lySeparatoren.html), [marginalia](https://ocr-d.de/en/gt-guidelines/trans/lyMarginalie.html), [initial](https://ocr-d.de/en/gt-guidelines/trans/lyInitiale.html), [table](https://ocr-d.de/en/gt-guidelines/trans/lyTabellen.html) -* Support for various image optimization operations: - * cropping (border detection), binarization, deskewing, dewarping, scaling, enhancing, resizing -* Text line segmentation to bounding boxes or polygons (contours) including for curved lines and vertical text -* Detection of reading order (left-to-right or right-to-left) -* Output in [PAGE-XML](https://github.com/PRImA-Research-Lab/PAGE-XML) -* [OCR-D](https://github.com/qurator-spk/eynollah#use-as-ocr-d-processor) interface +## Introduction +This tool performs document layout analysis (segmentation) from image data and returns the results as [PAGE-XML](https://github.com/PRImA-Research-Lab/PAGE-XML). -:warning: Development is currently focused on achieving the best possible quality of results for a wide variety of historical documents and therefore processing can be very slow. We aim to improve this, but contributions are welcome. +It can currently detect the following layout classes/elements: +* [Border](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_BorderType.html) +* [Textregion](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextRegionType.html) +* [Textline](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_TextLineType.html) +* [Image](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_ImageRegionType.html) +* [Separator](https://ocr-d.de/en/gt-guidelines/pagexml/pagecontent_xsd_Complex_Type_pc_SeparatorRegionType.html) +* [Marginalia](https://ocr-d.de/en/gt-guidelines/trans/lyMarginalie.html) +* [Initial (Drop Capital)](https://ocr-d.de/en/gt-guidelines/trans/lyInitiale.html) + +In addition, the tool can be used to detect the _[ReadingOrder](https://ocr-d.de/en/gt-guidelines/trans/lyLeserichtung.html)_ of regions. The final goal is to feed the output to an OCR model. + +The tool uses a combination of various models and heuristics (see flowchart below for the different stages and how they interact): +* [Border detection](https://github.com/qurator-spk/eynollah#border-detection) +* [Layout detection](https://github.com/qurator-spk/eynollah#layout-detection) +* [Textline detection](https://github.com/qurator-spk/eynollah#textline-detection) +* [Image enhancement](https://github.com/qurator-spk/eynollah#Image_enhancement) +* [Scale classification](https://github.com/qurator-spk/eynollah#Scale_classification) +* [Heuristic methods](https://https://github.com/qurator-spk/eynollah#heuristic-methods) + +The first three stages are based on [pixel-wise segmentation](https://github.com/qurator-spk/sbb_pixelwise_segmentation). + +![](https://user-images.githubusercontent.com/952378/100619946-1936f680-331e-11eb-9297-6e8b4cab3c16.png) + +## Border detection +For the purpose of text recognition (OCR) and in order to avoid noise being introduced from texts outside the printspace, one first needs to detect the border of the printed frame. This is done by a binary pixel-wise-segmentation model trained on a dataset of 2,000 documents where about 1,200 of them come from the [dhSegment](https://github.com/dhlab-epfl/dhSegment/) project (you can download the dataset from [here](https://github.com/dhlab-epfl/dhSegment/releases/download/v0.2/pages.zip)) and the remainder having been annotated in SBB. For border detection, the model needs to be fed with the whole image at once rather than separated in patches. + +## Layout detection +As a next step, text regions need to be identified by means of layout detection. Again a pixel-wise segmentation model was trained on 131 labeled images from the SBB digital collections, including some data augmentation. Since the target of this tool are historical documents, we consider as main region types text regions, separators, images, tables and background - each with their own subclasses, e.g. in the case of text regions, subclasses like header/heading, drop capital, main body text etc. While it would be desirable to detect and classify each of these classes in a granular way, there are also limitations due to having a suitably large and balanced training set. Accordingly, the current version of this tool is focussed on the main region types background, text region, image and separator. + +## Textline detection +In a subsequent step, binary pixel-wise segmentation is used again to classify pixels in a document that constitute textlines. For textline segmentation, a model was initially trained on documents with only one column/block of text and some augmentation with regard to scaling. By fine-tuning the parameters also for multi-column documents, additional training data was produced that resulted in a much more robust textline detection model. + +## Image enhancement +This is an image to image model which input was low quality of an image and label was actually the original image. For this one we did not have any GT, so we decreased the quality of documents in SBB and then feed them into model. + +## Scale classification +This is simply an image classifier which classifies images based on their scales or better to say based on their number of columns. + +## Heuristic methods +Some heuristic methods are also employed to further improve the model predictions: +* After border detection, the largest contour is determined by a bounding box, and the image cropped to these coordinates. +* For text region detection, the image is scaled up to make it easier for the model to detect background space between text regions. +* A minimum area is defined for text regions in relation to the overall image dimensions, so that very small regions that are noise can be filtered out. +* Deskewing is applied on the text region level (due to regions having different degrees of skew) in order to improve the textline segmentation result. +* After deskewing, a calculation of the pixel distribution on the X-axis allows the separation of textlines (foreground) and background pixels. +* Finally, using the derived coordinates, bounding boxes are determined for each textline. ## Installation -Python `3.8-3.11` with Tensorflow `<2.13` on Linux are currently supported. +`pip install .` or -For (limited) GPU support the CUDA toolkit needs to be installed. +`pip install . -e` for editable installation -You can either install from PyPI +Alternatively, you can also use `make` with these targets: -``` -pip install eynollah -``` +`make install` or -or clone the repository, enter it and install (editable) with +`make install-dev` for editable installation -``` -git clone git@github.com:qurator-spk/eynollah.git -cd eynollah; pip install -e . -``` +### Models -Alternatively, you can run `make install` or `make install-dev` for editable installation. +In order to run this tool you also need trained models. You can download our pretrained models from [qurator-data.de](https://qurator-data.de/eynollah/). -## Models -Pre-trained models can be downloaded from [qurator-data.de](https://qurator-data.de/eynollah/) or [huggingface](https://huggingface.co/SBB?search_models=eynollah). - -For documentation on methods and models, have a look at [`models.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/models.md). - -## Train -In case you want to train your own model with Eynollah, have a look at [`train.md`](https://github.com/qurator-spk/eynollah/tree/main/docs/train.md). +Alternatively, running `make models` will download and extract models to `$(PWD)/models_eynollah`. ## Usage -The command-line interface can be called like this: + +The basic command-line interface can be called like this: ```sh eynollah \ - -i | -di \ - -o \ - -m \ - [OPTIONS] +-i \ +-o \ +-m \ +-fl \ +-ae \ +-as \ +-cl \ +-si ``` -The following options can be used to further configure the processing: +The tool does accept and works better on original images (RGB format) than binarized images. -| option | description | -|-------------------|:-------------------------------------------------------------------------------| -| `-fl` | full layout analysis including all steps and segmentation classes | -| `-light` | lighter and faster but simpler method for main region detection and deskewing | -| `-tab` | apply table detection | -| `-ae` | apply enhancement (the resulting image is saved to the output directory) | -| `-as` | apply scaling | -| `-cl` | apply contour detection for curved text lines instead of bounding boxes | -| `-ib` | apply binarization (the resulting image is saved to the output directory) | -| `-ep` | enable plotting (MUST always be used with `-sl`, `-sd`, `-sa`, `-si` or `-ae`) | -| `-eoi` | extract only images to output directory (other processing will not be done) | -| `-ho` | ignore headers for reading order dectection | -| `-si ` | save image regions detected to this directory | -| `-sd ` | save deskewed image to this directory | -| `-sl ` | save layout prediction as plot to this directory | -| `-sp ` | save cropped page image to this directory | -| `-sa ` | save all (plot, enhanced/binary image, layout) to this directory | +### `--full-layout` vs `--no-full-layout` -If no option is set, the tool performs layout detection of main regions (background, text, images, separators and marginals). -The best output quality is produced when RGB images are used as input rather than greyscale or binarized images. +Here are the difference in elements detected depending on the `--full-layout`/`--no-full-layout` command line flags: -#### Use as OCR-D processor +| | `--full-layout` | `--no-full-layout` | +| --- | --- | --- | +| reading order | x | x | +| header regions | x | - | +| text regions | x | x | +| text regions / text line | x | x | +| drop-capitals | x | - | +| marginals | x | x | +| marginals / text line | x | x | +| image region | x | x | -Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) [processor](https://ocr-d.de/en/spec/cli), -formally described in [`ocrd-tool.json`](https://github.com/qurator-spk/eynollah/tree/main/src/eynollah/ocrd-tool.json). +### How to use -In this case, the source image file group with (preferably) RGB images should be used as input like this: +First, this model makes use of up to 9 trained models which are responsible for different operations like size detection, column classification, image enhancement, page extraction, main layout detection, full layout detection and textline detection.That does not mean that all 9 models are always required for every document. Based on the document characteristics and parameters specified, different scenarios can be applied. - ocrd-eynollah-segment -I OCR-D-IMG -O OCR-D-SEG -P models 2022-04-05 +* If none of the parameters is set to `true`, the tool will perform a layout detection of main regions (background, text, images, separators and marginals). An advantage of this tool is that it tries to extract main text regions separately as much as possible. +* If you set `-ae` (**a**llow image **e**nhancement) parameter to `true`, the tool will first check the ppi (pixel-per-inch) of the image and when it is less than 300, the tool will resize it and only then image enhancement will occur. Image enhancement can also take place without this option, but by setting this option to `true`, the layout xml data (e.g. coordinates) will be based on the resized and enhanced image instead of the original image. -If the input file group is PAGE-XML (from a previous OCR-D workflow step), Eynollah behaves as follows: -- existing regions are kept and ignored (i.e. in effect they might overlap segments from Eynollah results) -- existing annotation (and respective `AlternativeImage`s) are partially _ignored_: - - previous page frame detection (`cropped` images) - - previous derotation (`deskewed` images) - - previous thresholding (`binarized` images) -- if the page-level image nevertheless deviates from the original (`@imageFilename`) - (because some other preprocessing step was in effect like `denoised`), then - the output PAGE-XML will be based on that as new top-level (`@imageFilename`) +* For some documents, while the quality is good, their scale is very large, and the performance of tool decreases. In such cases you can set `-as` (**a**llow **s**caling) to `true`. With this option enabled, the tool will try to rescale the image and only then the layout detection process will begin. +* If you care about drop capitals (initials) and headings, you can set `-fl` (**f**ull **l**ayout) to `true`. With this setting, the tool can currently distinguish 7 document layout classes/elements. - ocrd-eynollah-segment -I OCR-D-XYZ -O OCR-D-SEG -P models 2022-04-05 +* In cases where the document includes curved headers or curved lines, rectangular bounding boxes for textlines will not be a great option. In such cases it is strongly recommended setting the flag `-cl` (**c**urved **l**ines) to `true` to find contours of curved lines instead of rectangular bounding boxes. Be advised that enabling this option increases the processing time of the tool. -Still, in general, it makes more sense to add other workflow steps **after** Eynollah. +* To crop and save image regions inside the document, set the parameter `-si` (**s**ave **i**mages) to true and provide a directory path to store the extracted images. -#### Additional documentation -Please check the [wiki](https://github.com/qurator-spk/eynollah/wiki). - -## How to cite -If you find this tool useful in your work, please consider citing our paper: - -```bibtex -@inproceedings{hip23rezanezhad, - title = {Document Layout Analysis with Deep Learning and Heuristics}, - author = {Rezanezhad, Vahid and Baierer, Konstantin and Gerber, Mike and Labusch, Kai and Neudecker, Clemens}, - booktitle = {Proceedings of the 7th International Workshop on Historical Document Imaging and Processing {HIP} 2023, - San José, CA, USA, August 25-26, 2023}, - publisher = {Association for Computing Machinery}, - address = {New York, NY, USA}, - year = {2023}, - pages = {73--78}, - url = {https://doi.org/10.1145/3604951.3605513} -} -``` +* This tool is actively being developed. If problems occur, or the performance does not meet your expectations, we welcome your feedback via [issues](https://github.com/qurator-spk/eynollah/issues). diff --git a/docs/models.md b/docs/models.md deleted file mode 100644 index ac563b0..0000000 --- a/docs/models.md +++ /dev/null @@ -1,147 +0,0 @@ -# Models documentation -This suite of 14 models presents a document layout analysis (DLA) system for historical documents implemented by -pixel-wise segmentation using a combination of a ResNet50 encoder with various U-Net decoders. In addition, heuristic -methods are applied to detect marginals and to determine the reading order of text regions. - -The detection and classification of multiple classes of layout elements such as headings, images, tables etc. as part of -DLA is required in order to extract and process them in subsequent steps. Altogether, the combination of image -detection, classification and segmentation on the wide variety that can be found in over 400 years of printed cultural -heritage makes this a very challenging task. Deep learning models are complemented with heuristics for the detection of -text lines, marginals, and reading order. Furthermore, an optional image enhancement step was added in case of documents -that either have insufficient pixel density and/or require scaling. Also, a column classifier for the analysis of -multi-column documents was added. With these additions, DLA performance was improved, and a high accuracy in the -prediction of the reading order is accomplished. - -Two Arabic/Persian terms form the name of the model suite: عين الله, which can be transcribed as "ain'allah" or -"eynollah"; it translates into English as "God's Eye" -- it sees (nearly) everything on the document image. - -See the flowchart below for the different stages and how they interact: - -![](https://user-images.githubusercontent.com/952378/100619946-1936f680-331e-11eb-9297-6e8b4cab3c16.png) - - -## Models - -### Image enhancement -Model card: [Image Enhancement](https://huggingface.co/SBB/eynollah-enhancement) - -This model addresses image resolution, specifically targeting documents with suboptimal resolution. In instances where -the detection of document layout exhibits inadequate performance, the proposed enhancement aims to significantly improve -the quality and clarity of the images, thus facilitating enhanced visual interpretation and analysis. - -### Page extraction / border detection -Model card: [Page Extraction/Border Detection](https://huggingface.co/SBB/eynollah-page-extraction) - -A problem that can negatively affect OCR are black margins around a page caused by document scanning. A deep learning -model helps to crop to the page borders by using a pixel-wise segmentation method. - -### Column classification -Model card: [Column Classification](https://huggingface.co/SBB/eynollah-column-classifier) - -This model is a trained classifier that recognizes the number of columns in a document by use of a training set with -manual classification of all documents into six classes with either one, two, three, four, five, or six and more columns -respectively. - -### Binarization -Model card: [Binarization](https://huggingface.co/SBB/eynollah-binarization) - -This model is designed to tackle the intricate task of document image binarization, which involves segmentation of the -image into white and black pixels. This process significantly contributes to the overall performance of the layout -models, particularly in scenarios where the documents are degraded or exhibit subpar quality. The robust binarization -capability of the model enables improved accuracy and reliability in subsequent layout analysis, thereby facilitating -enhanced document understanding and interpretation. - -### Main region detection -Model card: [Main Region Detection](https://huggingface.co/SBB/eynollah-main-regions) - -This model has employed a different set of labels, including an artificial class specifically designed to encompass the -text regions. The inclusion of this artificial class facilitates easier isolation of text regions by the model. This -approach grants the advantage of training the model using downscaled images, which in turn leads to faster predictions -during the inference phase. By incorporating this methodology, improved efficiency is achieved without compromising the -model's ability to accurately identify and classify text regions within documents. - -### Main region detection (with scaling augmentation) -Model card: [Main Region Detection (with scaling augmentation)](https://huggingface.co/SBB/eynollah-main-regions-aug-scaling) - -Utilizing scaling augmentation, this model leverages the capability to effectively segment elements of extremely high or -low scales within documents. By harnessing this technique, the tool gains a significant advantage in accurately -categorizing and isolating such elements, thereby enhancing its overall performance and enabling precise analysis of -documents with varying scale characteristics. - -### Main region detection (with rotation augmentation) -Model card: [Main Region Detection (with rotation augmentation)](https://huggingface.co/SBB/eynollah-main-regions-aug-rotation) - -This model takes advantage of rotation augmentation. This helps the tool to segment the vertical text regions in a -robust way. - -### Main region detection (ensembled) -Model card: [Main Region Detection (ensembled)](https://huggingface.co/SBB/eynollah-main-regions-ensembled) - -The robustness of this model is attained through an ensembling technique that combines the weights from various epochs. -By employing this approach, the model achieves a high level of resilience and stability, effectively leveraging the -strengths of multiple epochs to enhance its overall performance and deliver consistent and reliable results. - -### Full region detection (1,2-column documents) -Model card: [Full Region Detection (1,2-column documents)](https://huggingface.co/SBB/eynollah-full-regions-1column) - -This model deals with documents comprising of one and two columns. - -### Full region detection (3,n-column documents) -Model card: [Full Region Detection (3,n-column documents)](https://huggingface.co/SBB/eynollah-full-regions-3pluscolumn) - -This model is responsible for detecting headers and drop capitals in documents with three or more columns. - -### Textline detection -Model card: [Textline Detection](https://huggingface.co/SBB/eynollah-textline) - -The method for textline detection combines deep learning and heuristics. In the deep learning part, an image-to-image -model performs binary segmentation of the document into the classes textline vs. background. In the heuristics part, -bounding boxes or contours are derived from binary segmentation. - -Skewed documents can heavily affect textline detection accuracy, so robust deskewing is needed. But detecting textlines -with rectangle bounding boxes cannot deal with partially curved textlines. To address this, a functionality -specifically for documents with curved textlines was included. After finding the contour of a text region and its -corresponding textline segmentation, the text region is cut into smaller vertical straps. For each strap, its textline -segmentation is first deskewed and then the textlines are separated with the same heuristic method as for finding -textline bounding boxes. Later, the strap is rotated back into its original orientation. - -### Textline detection (light) -Model card: [Textline Detection Light (simpler but faster method)](https://huggingface.co/SBB/eynollah-textline_light) - -The method for textline detection combines deep learning and heuristics. In the deep learning part, an image-to-image -model performs binary segmentation of the document into the classes textline vs. background. In the heuristics part, -bounding boxes or contours are derived from binary segmentation. - -In the context of this textline model, a distinct labeling approach has been employed to ensure accurate predictions. -Specifically, an artificial bounding class has been incorporated alongside the textline classes. This strategic -inclusion effectively prevents any spurious connections between adjacent textlines during the prediction phase, thereby -enhancing the model's ability to accurately identify and delineate individual textlines within documents. This model -eliminates the need for additional heuristics in extracting textline contours. - -### Table detection -Model card: [Table Detection](https://huggingface.co/SBB/eynollah-tables) - -The objective of this model is to perform table segmentation in historical document images. Due to the pixel-wise -segmentation approach employed and the presence of traditional tables predominantly composed of text, the detection of -tables required the incorporation of heuristics to achieve reasonable performance. These heuristics were necessary to -effectively identify and delineate tables within the historical document images, ensuring accurate segmentation and -enabling subsequent analysis and interpretation. - -### Image detection -Model card: [Image Detection](https://huggingface.co/SBB/eynollah-image-extraction) - -This model is used for the task of illustration detection only. - -### Reading order detection -Model card: [Reading Order Detection]() - -TODO - -## Heuristic methods -Additionally, some heuristic methods are employed to further improve the model predictions: -* After border detection, the largest contour is determined by a bounding box, and the image cropped to these coordinates. -* For text region detection, the image is scaled up to make it easier for the model to detect background space between text regions. -* A minimum area is defined for text regions in relation to the overall image dimensions, so that very small regions that are noise can be filtered out. -* Deskewing is applied on the text region level (due to regions having different degrees of skew) in order to improve the textline segmentation result. -* After deskewing, a calculation of the pixel distribution on the X-axis allows the separation of textlines (foreground) and background pixels. -* Finally, using the derived coordinates, bounding boxes are determined for each textline. diff --git a/docs/train.md b/docs/train.md deleted file mode 100644 index 9f44a63..0000000 --- a/docs/train.md +++ /dev/null @@ -1,632 +0,0 @@ -# Training documentation -This aims to assist users in preparing training datasets, training models, and performing inference with trained models. -We cover various use cases including pixel-wise segmentation, image classification, image enhancement, and machine-based -reading order detection. For each use case, we provide guidance on how to generate the corresponding training dataset. - -The following three tasks can all be accomplished using the code in the -[`train`](https://github.com/qurator-spk/sbb_pixelwise_segmentation/tree/unifying-training-models) directory: - -* generate training dataset -* train a model -* inference with the trained model - -## Generate training dataset -The script `generate_gt_for_training.py` is used for generating training datasets. As the results of the following -command demonstrates, the dataset generator provides three different commands: - -`python generate_gt_for_training.py --help` - -These three commands are: - -* image-enhancement -* machine-based-reading-order -* pagexml2label - -### image-enhancement -Generating a training dataset for image enhancement is quite straightforward. All that is needed is a set of -high-resolution images. The training dataset can then be generated using the following command: - -`python generate_gt_for_training.py image-enhancement -dis "dir of high resolution images" -dois "dir where degraded -images will be written" -dols "dir where the corresponding high resolution image will be written as label" -scs -"degrading scales json file"` - -The scales JSON file is a dictionary with a key named 'scales' and values representing scales smaller than 1. Images are -downscaled based on these scales and then upscaled again to their original size. This process causes the images to lose -resolution at different scales. The degraded images are used as input images, and the original high-resolution images -serve as labels. The enhancement model can be trained with this generated dataset. The scales JSON file looks like this: - -```yaml -{ - "scales": [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9] -} -``` - -### machine-based-reading-order -For machine-based reading order, we aim to determine the reading priority between two sets of text regions. The model's -input is a three-channel image: the first and last channels contain information about each of the two text regions, -while the middle channel encodes prominent layout elements necessary for reading order, such as separators and headers. -To generate the training dataset, our script requires a page XML file that specifies the image layout with the correct -reading order. - -For output images, it is necessary to specify the width and height. Additionally, a minimum text region size can be set -to filter out regions smaller than this minimum size. This minimum size is defined as the ratio of the text region area -to the image area, with a default value of zero. To run the dataset generator, use the following command: - -`python generate_gt_for_training.py machine-based-reading-order -dx "dir of GT xml files" -domi "dir where output images -will be written" -docl "dir where the labels will be written" -ih "height" -iw "width" -min "min area ratio"` - -### pagexml2label -pagexml2label is designed to generate labels from GT page XML files for various pixel-wise segmentation use cases, -including 'layout,' 'textline,' 'printspace,' 'glyph,' and 'word' segmentation. -To train a pixel-wise segmentation model, we require images along with their corresponding labels. Our training script -expects a PNG image where each pixel corresponds to a label, represented by an integer. The background is always labeled -as zero, while other elements are assigned different integers. For instance, if we have ground truth data with four -elements including the background, the classes would be labeled as 0, 1, 2, and 3 respectively. - -In binary segmentation scenarios such as textline or page extraction, the background is encoded as 0, and the desired -element is automatically encoded as 1 in the PNG label. - -To specify the desired use case and the elements to be extracted in the PNG labels, a custom JSON file can be passed. -For example, in the case of 'textline' detection, the JSON file would resemble this: - -```yaml -{ -"use_case": "textline" -} -``` - -In the case of layout segmentation a custom config json file can look like this: - -```yaml -{ -"use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -} -``` - -A possible custom config json file for layout segmentation where the "printspace" is a class: - -```yaml -{ -"use_case": "layout", -"textregions":{"rest_as_paragraph":1 , "drop-capital": 1, "header":2, "heading":2, "marginalia":3}, -"imageregion":4, -"separatorregion":5, -"graphicregions" :{"rest_as_decoration":6 ,"stamp":7} -"printspace_as_class_in_layout" : 8 -} -``` - -For the layout use case, it is beneficial to first understand the structure of the page XML file and its elements. -In a given image, the annotations of elements are recorded in a page XML file, including their contours and classes. -For an image document, the known regions are 'textregion', 'separatorregion', 'imageregion', 'graphicregion', -'noiseregion', and 'tableregion'. - -Text regions and graphic regions also have their own specific types. The known types for text regions are 'paragraph', -'header', 'heading', 'marginalia', 'drop-capital', 'footnote', 'footnote-continued', 'signature-mark', 'page-number', -and 'catch-word'. The known types for graphic regions are 'handwritten-annotation', 'decoration', 'stamp', and -'signature'. -Since we don't know all types of text and graphic regions, unknown cases can arise. To handle these, we have defined -two additional types, "rest_as_paragraph" and "rest_as_decoration", to ensure that no unknown types are missed. -This way, users can extract all known types from the labels and be confident that no unknown types are overlooked. - -In the custom JSON file shown above, "header" and "heading" are extracted as the same class, while "marginalia" is shown -as a different class. All other text region types, including "drop-capital," are grouped into the same class. For the -graphic region, "stamp" has its own class, while all other types are classified together. "Image region" and "separator -region" are also present in the label. However, other regions like "noise region" and "table region" will not be -included in the label PNG file, even if they have information in the page XML files, as we chose not to include them. - -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will -be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just -to visualise the labels" "` - -We have also defined an artificial class that can be added to the boundary of text region types or text lines. This key -is called "artificial_class_on_boundary." If users want to apply this to certain text regions in the layout use case, -the example JSON config file should look like this: - -```yaml -{ - "use_case": "layout", - "textregions": { - "paragraph": 1, - "drop-capital": 1, - "header": 2, - "heading": 2, - "marginalia": 3 - }, - "imageregion": 4, - "separatorregion": 5, - "graphicregions": { - "rest_as_decoration": 6 - }, - "artificial_class_on_boundary": ["paragraph", "header", "heading", "marginalia"], - "artificial_class_label": 7 -} -``` - -This implies that the artificial class label, denoted by 7, will be present on PNG files and will only be added to the -elements labeled as "paragraph," "header," "heading," and "marginalia." - -For "textline", "word", and "glyph", the artificial class on the boundaries will be activated only if the -"artificial_class_label" key is specified in the config file. Its value should be set as 2 since these elements -represent binary cases. For example, if the background and textline are denoted as 0 and 1 respectively, then the -artificial class should be assigned the value 2. The example JSON config file should look like this for "textline" use -case: - -```yaml -{ - "use_case": "textline", - "artificial_class_label": 2 -} -``` - -If the coordinates of "PrintSpace" or "Border" are present in the page XML ground truth files, and the user wishes to -crop only the print space area, this can be achieved by activating the "-ps" argument. However, it should be noted that -in this scenario, since cropping will be applied to the label files, the directory of the original images must be -provided to ensure that they are cropped in sync with the labels. This ensures that the correct images and labels -required for training are obtained. The command should resemble the following: - -`python generate_gt_for_training.py pagexml2label -dx "dir of GT xml files" -do "dir where output label png files will -be written" -cfg "custom config json file" -to "output type which has 2d and 3d. 2d is used for training and 3d is just -to visualise the labels" -ps -di "dir where the org images are located" -doi "dir where the cropped output images will -be written" ` - -## Train a model -### classification - -For the classification use case, we haven't provided a ground truth generator, as it's unnecessary. For classification, -all we require is a training directory with subdirectories, each containing images of its respective classes. We need -separate directories for training and evaluation, and the class names (subdirectories) must be consistent across both -directories. Additionally, the class names should be specified in the config JSON file, as shown in the following -example. If, for instance, we aim to classify "apple" and "orange," with a total of 2 classes, the -"classification_classes_name" key in the config file should appear as follows: - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "classification", - "n_classes" : 2, - "n_epochs" : 10, - "input_height" : 448, - "input_width" : 448, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "f1_threshold_classification": 0.8, - "pretraining" : true, - "classification_classes_name" : {"0":"apple", "1":"orange"}, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── apple # directory of images for apple class - └── orange # directory of images for orange class -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── apple # directory of images for apple class - └── orange # directory of images for orange class - -``` - -The classification model can be trained using the following command line: - -`python train.py with config_classification.json` - -As evident in the example JSON file above, for classification, we utilize a "f1_threshold_classification" parameter. -This parameter is employed to gather all models with an evaluation f1 score surpassing this threshold. Subsequently, -an ensemble of these model weights is executed, and a model is saved in the output directory as "model_ens_avg". -Additionally, the weight of the best model based on the evaluation f1 score is saved as "model_best". - -### reading order -An example config json file for machine based reading order should be like this: - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "reading_order", - "n_classes" : 1, - "n_epochs" : 5, - "input_height" : 672, - "input_width" : 448, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "pretraining" : true, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── images # directory of images - └── labels # directory of labels -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── images # directory of images - └── labels # directory of labels -``` - -The classification model can be trained like the classification case command line. - -### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement - -#### Parameter configuration for segmentation or enhancement usecases -The following parameter configuration can be applied to all segmentation use cases and enhancements. The augmentation, -its sub-parameters, and continued training are defined only for segmentation use cases and enhancements, not for -classification and machine-based reading order, as you can see in their example config files. - -* backbone_type: For segmentation tasks (such as text line, binarization, and layout detection) and enhancement, we -* offer two backbone options: a "nontransformer" and a "transformer" backbone. For the "transformer" backbone, we first -* apply a CNN followed by a transformer. In contrast, the "nontransformer" backbone utilizes only a CNN ResNet-50. -* task : The task parameter can have values such as "segmentation", "enhancement", "classification", and "reading_order". -* patches: If you want to break input images into smaller patches (input size of the model) you need to set this -* parameter to ``true``. In the case that the model should see the image once, like page extraction, patches should be -* set to ``false``. -* n_batch: Number of batches at each iteration. -* n_classes: Number of classes. In the case of binary classification this should be 2. In the case of reading_order it -* should set to 1. And for the case of layout detection just the unique number of classes should be given. -* n_epochs: Number of epochs. -* input_height: This indicates the height of model's input. -* input_width: This indicates the width of model's input. -* weight_decay: Weight decay of l2 regularization of model layers. -* pretraining: Set to ``true`` to load pretrained weights of ResNet50 encoder. The downloaded weights should be saved -* in a folder named "pretrained_model" in the same directory of "train.py" script. -* augmentation: If you want to apply any kind of augmentation this parameter should first set to ``true``. -* flip_aug: If ``true``, different types of filp will be applied on image. Type of flips is given with "flip_index" parameter. -* blur_aug: If ``true``, different types of blurring will be applied on image. Type of blurrings is given with "blur_k" parameter. -* scaling: If ``true``, scaling will be applied on image. Scale of scaling is given with "scales" parameter. -* degrading: If ``true``, degrading will be applied to the image. The amount of degrading is defined with "degrade_scales" parameter. -* brightening: If ``true``, brightening will be applied to the image. The amount of brightening is defined with "brightness" parameter. -* rotation_not_90: If ``true``, rotation (not 90 degree) will be applied on image. Rotation angles are given with "thetha" parameter. -* rotation: If ``true``, 90 degree rotation will be applied on image. -* binarization: If ``true``,Otsu thresholding will be applied to augment the input data with binarized images. -* scaling_bluring: If ``true``, combination of scaling and blurring will be applied on image. -* scaling_binarization: If ``true``, combination of scaling and binarization will be applied on image. -* scaling_flip: If ``true``, combination of scaling and flip will be applied on image. -* flip_index: Type of flips. -* blur_k: Type of blurrings. -* scales: Scales of scaling. -* brightness: The amount of brightenings. -* thetha: Rotation angles. -* degrade_scales: The amount of degradings. -* continue_training: If ``true``, it means that you have already trained a model and you would like to continue the training. So it is needed to provide the dir of trained model with "dir_of_start_model" and index for naming the models. For example if you have already trained for 3 epochs then your last index is 2 and if you want to continue from model_1.h5, you can set ``index_start`` to 3 to start naming model with index 3. -* weighted_loss: If ``true``, this means that you want to apply weighted categorical_crossentropy as loss fucntion. Be carefull if you set to ``true``the parameter "is_loss_soft_dice" should be ``false`` -* data_is_provided: If you have already provided the input data you can set this to ``true``. Be sure that the train and eval data are in "dir_output". Since when once we provide training data we resize and augment them and then we write them in sub-directories train and eval in "dir_output". -* dir_train: This is the directory of "images" and "labels" (dir_train should include two subdirectories with names of images and labels ) for raw images and labels. Namely they are not prepared (not resized and not augmented) yet for training the model. When we run this tool these raw data will be transformed to suitable size needed for the model and they will be written in "dir_output" in train and eval directories. Each of train and eval include "images" and "labels" sub-directories. -* index_start: Starting index for saved models in the case that "continue_training" is ``true``. -* dir_of_start_model: Directory containing pretrained model to continue training the model in the case that "continue_training" is ``true``. -* transformer_num_patches_xy: Number of patches for vision transformer in x and y direction respectively. -* transformer_patchsize_x: Patch size of vision transformer patches in x direction. -* transformer_patchsize_y: Patch size of vision transformer patches in y direction. -* transformer_projection_dim: Transformer projection dimension. Default value is 64. -* transformer_mlp_head_units: Transformer Multilayer Perceptron (MLP) head units. Default value is [128, 64]. -* transformer_layers: transformer layers. Default value is 8. -* transformer_num_heads: Transformer number of heads. Default value is 4. -* transformer_cnn_first: We have two types of vision transformers. In one type, a CNN is applied first, followed by a transformer. In the other type, this order is reversed. If transformer_cnn_first is true, it means the CNN will be applied before the transformer. Default value is true. - -In the case of segmentation and enhancement the train and evaluation directory should be as following. - -The "dir_train" should be like this: - -``` -. -└── train # train directory - ├── images # directory of images - └── labels # directory of labels -``` - -And the "dir_eval" the same structure as train directory: - -``` -. -└── eval # evaluation directory - ├── images # directory of images - └── labels # directory of labels -``` - -After configuring the JSON file for segmentation or enhancement, training can be initiated by running the following -command, similar to the process for classification and reading order: - -`python train.py with config_classification.json` - -#### Binarization -An example config json file for binarization can be like this: - -```yaml -{ - "backbone_type" : "transformer", - "task": "binarization", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 224, - "input_width" : 672, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "transformer_num_patches_xy": [7, 7], - "transformer_patchsize_x": 3, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 192, - "transformer_mlp_head_units": [128, 64], - "transformer_layers": 8, - "transformer_num_heads": 4, - "transformer_cnn_first": true, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -#### Textline - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "segmentation", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -#### Enhancement - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "enhancement", - "n_classes" : 3, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 4, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -It's important to mention that the value of n_classes for enhancement should be 3, as the model's output is a 3-channel -image. - -#### Page extraction - -```yaml -{ - "backbone_type" : "nontransformer", - "task": "segmentation", - "n_classes" : 2, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : false, - "pretraining" : true, - "augmentation" : false, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` - -For page segmentation (or printspace or border segmentation), the model needs to view the input image in its entirety, -hence the patches parameter should be set to false. - -#### layout segmentation -An example config json file for layout segmentation with 5 classes (including background) can be like this: - -```yaml -{ - "backbone_type" : "transformer", - "task": "segmentation", - "n_classes" : 5, - "n_epochs" : 4, - "input_height" : 448, - "input_width" : 224, - "weight_decay" : 1e-6, - "n_batch" : 1, - "learning_rate": 1e-4, - "patches" : true, - "pretraining" : true, - "augmentation" : true, - "flip_aug" : false, - "blur_aug" : false, - "scaling" : true, - "degrading": false, - "brightening": false, - "binarization" : false, - "scaling_bluring" : false, - "scaling_binarization" : false, - "scaling_flip" : false, - "rotation": false, - "rotation_not_90": false, - "transformer_num_patches_xy": [7, 14], - "transformer_patchsize_x": 1, - "transformer_patchsize_y": 1, - "transformer_projection_dim": 64, - "transformer_mlp_head_units": [128, 64], - "transformer_layers": 8, - "transformer_num_heads": 4, - "transformer_cnn_first": true, - "blur_k" : ["blur","guass","median"], - "scales" : [0.6, 0.7, 0.8, 0.9, 1.1, 1.2, 1.4], - "brightness" : [1.3, 1.5, 1.7, 2], - "degrade_scales" : [0.2, 0.4], - "flip_index" : [0, 1, -1], - "thetha" : [10, -10], - "continue_training": false, - "index_start" : 0, - "dir_of_start_model" : " ", - "weighted_loss": false, - "is_loss_soft_dice": false, - "data_is_provided": false, - "dir_train": "./train", - "dir_eval": "./eval", - "dir_output": "./output" -} -``` -## Inference with the trained model - -### classification -For conducting inference with a trained model, you simply need to execute the following command line, specifying the -directory of the model and the image on which to perform inference: - -`python inference.py -m "model dir" -i "image" ` - -This will straightforwardly return the class of the image. - -### machine based reading order -To infer the reading order using a reading order model, we need a page XML file containing layout information but -without the reading order. We simply need to provide the model directory, the XML file, and the output directory. -The new XML file with the added reading order will be written to the output directory with the same name. -We need to run: - -`python inference.py -m "model dir" -xml "page xml file" -o "output dir to write new xml with reading order" ` - -### Segmentation (Textline, Binarization, Page extraction and layout) and enhancement -For conducting inference with a trained model for segmentation and enhancement you need to run the following command -line: - -`python inference.py -m "model dir" -i "image" -p -s "output image" ` - -Note that in the case of page extraction the -p flag is not needed. - -For segmentation or binarization tasks, if a ground truth (GT) label is available, the IoU evaluation metric can be -calculated for the output. To do this, you need to provide the GT label using the argument -gt. diff --git a/docs/usage.md b/docs/usage.md deleted file mode 100644 index da164de..0000000 --- a/docs/usage.md +++ /dev/null @@ -1,92 +0,0 @@ -# Usage documentation -The command-line interface can be called like this: - -```sh -eynollah \ - -i | -di \ - -o \ - -m \ - [OPTIONS] -``` - -## Processing options -The following options can be used to further configure the processing: - -| option | description | -|-------------------|:-------------------------------------------------------------------------------| -| `-fl` | full layout analysis including all steps and segmentation classes | -| `-light` | lighter and faster but simpler method for main region detection and deskewing | -| `-tab` | apply table detection | -| `-ae` | apply enhancement (the resulting image is saved to the output directory) | -| `-as` | apply scaling | -| `-cl` | apply contour detection for curved text lines instead of bounding boxes | -| `-ib` | apply binarization (the resulting image is saved to the output directory) | -| `-ep` | enable plotting (MUST always be used with `-sl`, `-sd`, `-sa`, `-si` or `-ae`) | -| `-eoi` | extract only images to output directory (other processing will not be done) | -| `-ho` | ignore headers for reading order dectection | -| `-si ` | save image regions detected to this directory | -| `-sd ` | save deskewed image to this directory | -| `-sl ` | save layout prediction as plot to this directory | -| `-sp ` | save cropped page image to this directory | -| `-sa ` | save all (plot, enhanced/binary image, layout) to this directory | - -If no option is set, the tool performs detection of main regions (background, text, images, separators and marginals). - -### `--full-layout` vs `--no-full-layout` - -Here are the difference in elements detected depending on the `--full-layout`/`--no-full-layout` command line flags: - -| | `--full-layout` | `--no-full-layout` | -|--------------------------|-----------------|--------------------| -| reading order | x | x | -| header regions | x | - | -| text regions | x | x | -| text regions / text line | x | x | -| drop-capitals | x | - | -| marginals | x | x | -| marginals / text line | x | x | -| image region | x | x | - -## Use as OCR-D processor -Eynollah ships with a CLI interface to be used as [OCR-D](https://ocr-d.de) processor that is described in -[`ocrd-tool.json`](https://github.com/qurator-spk/eynollah/tree/main/src/eynollah/ocrd-tool.json). - -The source image file group with (preferably) RGB images should be used as input for Eynollah like this: - -``` -ocrd-eynollah-segment -I OCR-D-IMG -O SEG-LINE -P models -``` - -Any image referenced by `@imageFilename` in PAGE-XML is passed on directly to Eynollah as a processor, so that e.g. - -``` -ocrd-eynollah-segment -I OCR-D-IMG-BIN -O SEG-LINE -P models -``` - -uses the original (RGB) image despite any binarization that may have occured in previous OCR-D processing steps. - -## Use with Docker -TODO - -## Hints -* The best output quality is produced when RGB images are used as input rather than greyscale or binarized images. -* If none of the parameters is set to `true`, the tool will perform a layout detection of main regions (background, -text, images, separators and marginals). An advantage of this tool is that it tries to extract main text regions -separately as much as possible. -* If you set `-ae` (**a**llow image **e**nhancement) parameter to `true`, the tool will first check the ppi -(pixel-per-inch) of the image and when it is less than 300, the tool will resize it and only then image enhancement will -occur. Image enhancement can also take place without this option, but by setting this option to `true`, the layout xml -data (e.g. coordinates) will be based on the resized and enhanced image instead of the original image. -* For some documents, while the quality is good, their scale is very large, and the performance of tool decreases. In -such cases you can set `-as` (**a**llow **s**caling) to `true`. With this option enabled, the tool will try to rescale -the image and only then the layout detection process will begin. -* If you care about drop capitals (initials) and headings, you can set `-fl` (**f**ull **l**ayout) to `true`. With this -setting, the tool can currently distinguish 7 document layout classes/elements. -* In cases where the document includes curved headers or curved lines, rectangular bounding boxes for textlines will not -be a great option. In such cases it is strongly recommended setting the flag `-cl` (**c**urved **l**ines) to `true` to -find contours of curved lines instead of rectangular bounding boxes. Be advised that enabling this option increases the -processing time of the tool. -* To crop and save image regions inside the document, set the parameter `-si` (**s**ave **i**mages) to true and provide -a directory path to store the extracted images. -* To extract only images from a document, set the parameter `-eoi` (**e**xtract **o**nly **i**mages). Choosing this -option disables any other processing. To save the cropped images add `-ep` and `-si`. diff --git a/ocrd-tool.json b/ocrd-tool.json index 711a192..5c48493 120000 --- a/ocrd-tool.json +++ b/ocrd-tool.json @@ -1 +1 @@ -src/eynollah/ocrd-tool.json \ No newline at end of file +qurator/eynollah/ocrd-tool.json \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 4da39ef..0000000 --- a/pyproject.toml +++ /dev/null @@ -1,53 +0,0 @@ -[build-system] -requires = ["setuptools>=61.0", "wheel", "setuptools-ocrd"] - -[project] -name = "eynollah" -authors = [ - {name = "Vahid Rezanezhad"}, - {name = "Staatsbibliothek zu Berlin - Preußischer Kulturbesitz"}, -] -description = "Document Layout Analysis" -readme = "README.md" -license.file = "LICENSE" -requires-python = ">=3.8" -keywords = ["document layout analysis", "image segmentation"] - -dynamic = ["dependencies", "version"] - -classifiers = [ - "Development Status :: 4 - Beta", - "Environment :: Console", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Topic :: Scientific/Engineering :: Image Processing", -] - -[project.optional-dependencies] -OCR = ["torch <= 2.0.1", "transformers <= 4.30.2"] -plotting = ["matplotlib"] - -[project.scripts] -eynollah = "eynollah.cli:main" -ocrd-eynollah-segment = "eynollah.ocrd_cli:main" -ocrd-sbb-binarize = "eynollah.ocrd_cli_binarization:main" - -[project.urls] -Homepage = "https://github.com/qurator-spk/eynollah" -Repository = "https://github.com/qurator-spk/eynollah.git" - -[tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} -optional-dependencies.test = {file = ["requirements-test.txt"]} - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -"*" = ["*.json", '*.yml', '*.xml', '*.xsd'] - -[tool.coverage.run] -branch = true -source = ["eynollah"] diff --git a/src/eynollah/__init__.py b/qurator/.gitkeep similarity index 100% rename from src/eynollah/__init__.py rename to qurator/.gitkeep diff --git a/qurator/__init__.py b/qurator/__init__.py new file mode 100644 index 0000000..5284146 --- /dev/null +++ b/qurator/__init__.py @@ -0,0 +1 @@ +__import__("pkg_resources").declare_namespace(__name__) diff --git a/qurator/eynollah/__init__.py b/qurator/eynollah/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/qurator/eynollah/__init__.py @@ -0,0 +1 @@ + diff --git a/qurator/eynollah/cli.py b/qurator/eynollah/cli.py new file mode 100644 index 0000000..a895b0d --- /dev/null +++ b/qurator/eynollah/cli.py @@ -0,0 +1,146 @@ +import sys +import click +from ocrd_utils import initLogging, setOverrideLogLevel +from qurator.eynollah.eynollah import Eynollah + + +@click.command() +@click.option( + "--image", + "-i", + help="image filename", + type=click.Path(exists=True, dir_okay=False), + required=True, +) +@click.option( + "--out", + "-o", + help="directory to write output xml data", + type=click.Path(exists=True, file_okay=False), + required=True, +) +@click.option( + "--model", + "-m", + help="directory of models", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_images", + "-si", + help="if a directory is given, images in documents will be cropped and saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_layout", + "-sl", + help="if a directory is given, plot of layout will be saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_deskewed", + "-sd", + help="if a directory is given, deskewed image will be saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--save_all", + "-sa", + help="if a directory is given, all plots needed for documentation will be saved there", + type=click.Path(exists=True, file_okay=False), +) +@click.option( + "--enable-plotting/--disable-plotting", + "-ep/-noep", + is_flag=True, + help="If set, will plot intermediary files and images", +) +@click.option( + "--allow-enhancement/--no-allow-enhancement", + "-ae/-noae", + is_flag=True, + help="if this parameter set to true, this tool would check that input image need resizing and enhancement or not. If so output of resized and enhanced image and corresponding layout data will be written in out directory", +) +@click.option( + "--curved-line/--no-curvedline", + "-cl/-nocl", + is_flag=True, + help="if this parameter set to true, this tool will try to return contoure of textlines instead of rectabgle bounding box of textline. This should be taken into account that with this option the tool need more time to do process.", +) +@click.option( + "--full-layout/--no-full-layout", + "-fl/-nofl", + is_flag=True, + help="if this parameter set to true, this tool will try to return all elements of layout.", +) +@click.option( + "--input_binary/--input-RGB", + "-ib/-irgb", + is_flag=True, + help="in general, eynollah uses RGB as input but if the input document is strongly dark, bright or for any other reason you can turn binarized input on. This option does not mean that you have to provide a binary image, otherwise this means that the tool itself will binarized the RGB input document.", +) +@click.option( + "--allow_scaling/--no-allow-scaling", + "-as/-noas", + is_flag=True, + help="if this parameter set to true, this tool would check the scale and if needed it will scale it to perform better layout detection", +) +@click.option( + "--headers-off/--headers-on", + "-ho/-noho", + is_flag=True, + help="if this parameter set to true, this tool would ignore headers role in reading order", +) +@click.option( + "--log-level", + "-l", + type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), + help="Override log level globally to this", +) +def main( + image, + out, + model, + save_images, + save_layout, + save_deskewed, + save_all, + enable_plotting, + allow_enhancement, + curved_line, + full_layout, + input_binary, + allow_scaling, + headers_off, + log_level +): + if log_level: + setOverrideLogLevel(log_level) + initLogging() + if not enable_plotting and (save_layout or save_deskewed or save_all or save_images): + print("Error: You used one of -sl, -sd, -sa or -si but did not enable plotting with -ep") + sys.exit(1) + elif enable_plotting and not (save_layout or save_deskewed or save_all or save_images): + print("Error: You used -ep to enable plotting but set none of -sl, -sd, -sa or -si") + sys.exit(1) + eynollah = Eynollah( + image_filename=image, + dir_out=out, + dir_models=model, + dir_of_cropped_images=save_images, + dir_of_layout=save_layout, + dir_of_deskewed=save_deskewed, + dir_of_all=save_all, + enable_plotting=enable_plotting, + allow_enhancement=allow_enhancement, + curved_line=curved_line, + full_layout=full_layout, + input_binary=input_binary, + allow_scaling=allow_scaling, + headers_off=headers_off, + ) + pcgts = eynollah.run() + eynollah.writer.write_pagexml(pcgts) + +if __name__ == "__main__": + main() diff --git a/qurator/eynollah/eynollah.py b/qurator/eynollah/eynollah.py new file mode 100644 index 0000000..2b8b97e --- /dev/null +++ b/qurator/eynollah/eynollah.py @@ -0,0 +1,2086 @@ +# pylint: disable=no-member,invalid-name,line-too-long,missing-function-docstring,missing-class-docstring,too-many-branches +# pylint: disable=too-many-locals,wrong-import-position,too-many-lines,too-many-statements,chained-comparison,fixme,broad-except,c-extension-no-member +# pylint: disable=too-many-public-methods,too-many-arguments,too-many-instance-attributes,too-many-public-methods, +# pylint: disable=consider-using-enumerate +""" +tool to extract table form data from alto xml data +""" + +import math +import os +import sys +import time +import warnings +from pathlib import Path +from multiprocessing import Process, Queue, cpu_count +import gc +from ocrd_utils import getLogger +import cv2 +import numpy as np +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" +stderr = sys.stderr +sys.stderr = open(os.devnull, "w") +from keras import backend as K +from keras.models import load_model +sys.stderr = stderr +import tensorflow as tf +tf.get_logger().setLevel("ERROR") +warnings.filterwarnings("ignore") + + +from .utils.contour import ( + filter_contours_area_of_image, + find_contours_mean_y_diff, + find_new_features_of_contours, + get_text_region_boxes_by_given_contours, + get_textregion_contours_in_org_image, + return_contours_of_image, + return_contours_of_interested_region, + return_contours_of_interested_region_by_min_size, + return_contours_of_interested_textline, + return_parent_contours, +) +from .utils.rotate import ( + rotate_image, + rotation_not_90_func, + rotation_not_90_func_full_layout) +from .utils.separate_lines import ( + textline_contours_postprocessing, + separate_lines_new2, + return_deskew_slop) +from .utils.drop_capitals import ( + adhere_drop_capital_region_into_corresponding_textline, + filter_small_drop_capitals_from_no_patch_layout) +from .utils.marginals import get_marginals +from .utils.resize import resize_image +from .utils import ( + boosting_headers_by_longshot_region_segmentation, + crop_image_inside_box, + find_num_col, + otsu_copy_binary, + put_drop_out_from_only_drop_model, + putt_bb_of_drop_capitals_of_model_in_patches_in_layout, + check_any_text_region_in_model_one_is_main_or_header, + small_textlines_to_parent_adherence2, + order_of_regions, + find_number_of_columns_in_document, + return_boxes_of_images_by_order_of_reading_new) +from .utils.pil_cv2 import check_dpi, pil2cv +from .utils.xml import order_and_id_of_texts +from .plot import EynollahPlotter +from .writer import EynollahXmlWriter + +SLOPE_THRESHOLD = 0.13 +RATIO_OF_TWO_MODEL_THRESHOLD = 95.50 #98.45: +DPI_THRESHOLD = 298 +MAX_SLOPE = 999 +KERNEL = np.ones((5, 5), np.uint8) + +class Eynollah: + def __init__( + self, + dir_models, + image_filename, + image_pil=None, + image_filename_stem=None, + dir_out=None, + dir_of_cropped_images=None, + dir_of_layout=None, + dir_of_deskewed=None, + dir_of_all=None, + enable_plotting=False, + allow_enhancement=False, + curved_line=False, + full_layout=False, + input_binary=False, + allow_scaling=False, + headers_off=False, + override_dpi=None, + logger=None, + pcgts=None, + ): + if image_pil: + self._imgs = self._cache_images(image_pil=image_pil) + else: + self._imgs = self._cache_images(image_filename=image_filename) + if override_dpi: + self.dpi = override_dpi + self.image_filename = image_filename + self.dir_out = dir_out + self.allow_enhancement = allow_enhancement + self.curved_line = curved_line + self.full_layout = full_layout + self.input_binary = input_binary + self.allow_scaling = allow_scaling + self.headers_off = headers_off + self.plotter = None if not enable_plotting else EynollahPlotter( + dir_of_all=dir_of_all, + dir_of_deskewed=dir_of_deskewed, + dir_of_cropped_images=dir_of_cropped_images, + dir_of_layout=dir_of_layout, + image_filename_stem=Path(Path(image_filename).name).stem) + self.writer = EynollahXmlWriter( + dir_out=self.dir_out, + image_filename=self.image_filename, + curved_line=self.curved_line, + pcgts=pcgts) + self.logger = logger if logger else getLogger('eynollah') + self.dir_models = dir_models + + self.model_dir_of_enhancement = dir_models + "/model_enhancement.h5" + self.model_dir_of_binarization = dir_models + "/model_bin_sbb_ens.h5" + self.model_dir_of_col_classifier = dir_models + "/model_scale_classifier.h5" + self.model_region_dir_p = dir_models + "/model_main_covid19_lr5-5_scale_1_1_great.h5" + self.model_region_dir_p2 = dir_models + "/model_main_home_corona3_rot.h5" + self.model_region_dir_fully_np = dir_models + "/model_no_patches_class0_30eopch.h5" + self.model_region_dir_fully = dir_models + "/model_3up_new_good_no_augmentation.h5" + self.model_page_dir = dir_models + "/model_page_mixed_best.h5" + self.model_region_dir_p_ens = dir_models + "/model_ensemble_s.h5" + self.model_textline_dir = dir_models + "/model_textline_newspapers.h5" + + def _cache_images(self, image_filename=None, image_pil=None): + ret = {} + if image_filename: + ret['img'] = cv2.imread(image_filename) + self.dpi = check_dpi(image_filename) + else: + ret['img'] = pil2cv(image_pil) + self.dpi = check_dpi(image_pil) + ret['img_grayscale'] = cv2.cvtColor(ret['img'], cv2.COLOR_BGR2GRAY) + for prefix in ('', '_grayscale'): + ret[f'img{prefix}_uint8'] = ret[f'img{prefix}'].astype(np.uint8) + return ret + + def imread(self, grayscale=False, uint8=True): + key = 'img' + if grayscale: + key += '_grayscale' + if uint8: + key += '_uint8' + return self._imgs[key].copy() + + def isNaN(self, num): + return num != num + + + def predict_enhancement(self, img): + self.logger.debug("enter predict_enhancement") + model_enhancement, session_enhancement = self.start_new_session_and_model(self.model_dir_of_enhancement) + + img_height_model = model_enhancement.layers[len(model_enhancement.layers) - 1].output_shape[1] + img_width_model = model_enhancement.layers[len(model_enhancement.layers) - 1].output_shape[2] + if img.shape[0] < img_height_model: + img = cv2.resize(img, (img.shape[1], img_width_model), interpolation=cv2.INTER_NEAREST) + + if img.shape[1] < img_width_model: + img = cv2.resize(img, (img_height_model, img.shape[0]), interpolation=cv2.INTER_NEAREST) + margin = int(0 * img_width_model) + width_mid = img_width_model - 2 * margin + height_mid = img_height_model - 2 * margin + img = img / float(255.0) + + img_h = img.shape[0] + img_w = img.shape[1] + + prediction_true = np.zeros((img_h, img_w, 3)) + nxf = img_w / float(width_mid) + nyf = img_h / float(height_mid) + + nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) + nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + + for i in range(nxf): + for j in range(nyf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + else: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + if j == 0: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + else: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - img_width_model + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - img_height_model + + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + label_p_pred = model_enhancement.predict(img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2])) + + seg = label_p_pred[0, :, :, :] + seg = seg * 255 + + if i == 0 and j == 0: + seg = seg[0 : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg + elif i == nxf - 1 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - 0] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0, :] = seg + elif i == 0 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin, :] = seg + elif i == nxf - 1 and j == 0: + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - 0] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg + elif i == 0 and j != 0 and j != nyf - 1: + seg = seg[margin : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg + elif i == nxf - 1 and j != 0 and j != nyf - 1: + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - 0] + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg + elif i != 0 and i != nxf - 1 and j == 0: + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - margin] + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg + elif i != 0 and i != nxf - 1 and j == nyf - 1: + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin, :] = seg + else: + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - margin] + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg + + prediction_true = prediction_true.astype(int) + session_enhancement.close() + del model_enhancement + del session_enhancement + gc.collect() + + return prediction_true + + def calculate_width_height_by_columns(self, img, num_col, width_early, label_p_pred): + self.logger.debug("enter calculate_width_height_by_columns") + if num_col == 1 and width_early < 1100: + img_w_new = 2000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2000) + elif num_col == 1 and width_early >= 2500: + img_w_new = 2000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2000) + elif num_col == 1 and width_early >= 1100 and width_early < 2500: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + elif num_col == 2 and width_early < 2000: + img_w_new = 2400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2400) + elif num_col == 2 and width_early >= 3500: + img_w_new = 2400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 2400) + elif num_col == 2 and width_early >= 2000 and width_early < 3500: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + elif num_col == 3 and width_early < 2000: + img_w_new = 3000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 3000) + elif num_col == 3 and width_early >= 4000: + img_w_new = 3000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 3000) + elif num_col == 3 and width_early >= 2000 and width_early < 4000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + elif num_col == 4 and width_early < 2500: + img_w_new = 4000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 4000) + elif num_col == 4 and width_early >= 5000: + img_w_new = 4000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 4000) + elif num_col == 4 and width_early >= 2500 and width_early < 5000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + elif num_col == 5 and width_early < 3700: + img_w_new = 5000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 5000) + elif num_col == 5 and width_early >= 7000: + img_w_new = 5000 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 5000) + elif num_col == 5 and width_early >= 3700 and width_early < 7000: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + elif num_col == 6 and width_early < 4500: + img_w_new = 6500 # 5400 + img_h_new = int(img.shape[0] / float(img.shape[1]) * 6500) + else: + img_w_new = width_early + img_h_new = int(img.shape[0] / float(img.shape[1]) * width_early) + + if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: + img_new = np.copy(img) + num_column_is_classified = False + else: + img_new = resize_image(img, img_h_new, img_w_new) + num_column_is_classified = True + + return img_new, num_column_is_classified + + def resize_image_with_column_classifier(self, is_image_enhanced, img_bin): + self.logger.debug("enter resize_image_with_column_classifier") + if self.input_binary: + img = np.copy(img_bin) + else: + img = self.imread() + + _, page_coord = self.early_page_for_num_of_column_classification(img) + model_num_classifier, session_col_classifier = self.start_new_session_and_model(self.model_dir_of_col_classifier) + if self.input_binary: + img_in = np.copy(img) + img_in = img_in / 255.0 + width_early = img_in.shape[1] + img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = img_in.reshape(1, 448, 448, 3) + else: + img_1ch = self.imread(grayscale=True, uint8=False) + width_early = img_1ch.shape[1] + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + # plt.imshow(img_1ch) + # plt.show() + img_1ch = img_1ch / 255.0 + + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + label_p_pred = model_num_classifier.predict(img_in) + num_col = np.argmax(label_p_pred[0]) + 1 + + self.logger.info("Found %s columns (%s)", num_col, label_p_pred) + + session_col_classifier.close() + + del model_num_classifier + del session_col_classifier + + K.clear_session() + gc.collect() + + + + img_new, _ = self.calculate_width_height_by_columns(img, num_col, width_early, label_p_pred) + + if img_new.shape[1] > img.shape[1]: + img_new = self.predict_enhancement(img_new) + is_image_enhanced = True + + return img, img_new, is_image_enhanced + + def resize_and_enhance_image_with_column_classifier(self): + self.logger.debug("enter resize_and_enhance_image_with_column_classifier") + dpi = self.dpi + self.logger.info("Detected %s DPI", dpi) + if self.input_binary: + img = self.imread() + model_bin, session_bin = self.start_new_session_and_model(self.model_dir_of_binarization) + prediction_bin = self.do_prediction(True, img, model_bin) + + prediction_bin=prediction_bin[:,:,0] + prediction_bin = (prediction_bin[:,:]==0)*1 + prediction_bin = prediction_bin*255 + + prediction_bin =np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) + + session_bin.close() + del model_bin + del session_bin + gc.collect() + + prediction_bin = prediction_bin.astype(np.uint8) + img= np.copy(prediction_bin) + img_bin = np.copy(prediction_bin) + else: + img = self.imread() + img_bin = None + + _, page_coord = self.early_page_for_num_of_column_classification(img_bin) + model_num_classifier, session_col_classifier = self.start_new_session_and_model(self.model_dir_of_col_classifier) + + if self.input_binary: + img_in = np.copy(img) + width_early = img_in.shape[1] + img_in = img_in / 255.0 + img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = img_in.reshape(1, 448, 448, 3) + else: + img_1ch = self.imread(grayscale=True) + width_early = img_1ch.shape[1] + img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + + img_1ch = img_1ch / 255.0 + img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) + img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) + img_in[0, :, :, 0] = img_1ch[:, :] + img_in[0, :, :, 1] = img_1ch[:, :] + img_in[0, :, :, 2] = img_1ch[:, :] + + + + label_p_pred = model_num_classifier.predict(img_in) + num_col = np.argmax(label_p_pred[0]) + 1 + self.logger.info("Found %s columns (%s)", num_col, label_p_pred) + session_col_classifier.close() + K.clear_session() + + if dpi < DPI_THRESHOLD: + img_new, num_column_is_classified = self.calculate_width_height_by_columns(img, num_col, width_early, label_p_pred) + image_res = self.predict_enhancement(img_new) + is_image_enhanced = True + else: + is_image_enhanced = False + num_column_is_classified = True + image_res = np.copy(img) + + session_col_classifier.close() + + + self.logger.debug("exit resize_and_enhance_image_with_column_classifier") + return is_image_enhanced, img, image_res, num_col, num_column_is_classified, img_bin + + # pylint: disable=attribute-defined-outside-init + def get_image_and_scales(self, img_org, img_res, scale): + self.logger.debug("enter get_image_and_scales") + self.image = np.copy(img_res) + self.image_org = np.copy(img_org) + self.height_org = self.image.shape[0] + self.width_org = self.image.shape[1] + + self.img_hight_int = int(self.image.shape[0] * scale) + self.img_width_int = int(self.image.shape[1] * scale) + self.scale_y = self.img_hight_int / float(self.image.shape[0]) + self.scale_x = self.img_width_int / float(self.image.shape[1]) + + self.image = resize_image(self.image, self.img_hight_int, self.img_width_int) + + # Also set for the plotter + if self.plotter: + self.plotter.image_org = self.image_org + self.plotter.scale_y = self.scale_y + self.plotter.scale_x = self.scale_x + # Also set for the writer + self.writer.image_org = self.image_org + self.writer.scale_y = self.scale_y + self.writer.scale_x = self.scale_x + self.writer.height_org = self.height_org + self.writer.width_org = self.width_org + + def get_image_and_scales_after_enhancing(self, img_org, img_res): + self.logger.debug("enter get_image_and_scales_after_enhancing") + self.image = np.copy(img_res) + self.image = self.image.astype(np.uint8) + self.image_org = np.copy(img_org) + self.height_org = self.image_org.shape[0] + self.width_org = self.image_org.shape[1] + + self.scale_y = img_res.shape[0] / float(self.image_org.shape[0]) + self.scale_x = img_res.shape[1] / float(self.image_org.shape[1]) + + # Also set for the plotter + if self.plotter: + self.plotter.image_org = self.image_org + self.plotter.scale_y = self.scale_y + self.plotter.scale_x = self.scale_x + # Also set for the writer + self.writer.image_org = self.image_org + self.writer.scale_y = self.scale_y + self.writer.scale_x = self.scale_x + self.writer.height_org = self.height_org + self.writer.width_org = self.width_org + + def start_new_session_and_model_old(self, model_dir): + self.logger.debug("enter start_new_session_and_model (model_dir=%s)", model_dir) + config = tf.ConfigProto() + config.gpu_options.allow_growth = True + + session = tf.InteractiveSession() + model = load_model(model_dir, compile=False) + + return model, session + + + def start_new_session_and_model(self, model_dir): + self.logger.debug("enter start_new_session_and_model (model_dir=%s)", model_dir) + gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) + #gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=7.7, allow_growth=True) + session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) + model = load_model(model_dir, compile=False) + + return model, session + + def do_prediction(self, patches, img, model, marginal_of_patch_percent=0.1): + self.logger.debug("enter do_prediction") + + img_height_model = model.layers[len(model.layers) - 1].output_shape[1] + img_width_model = model.layers[len(model.layers) - 1].output_shape[2] + + if not patches: + img_h_page = img.shape[0] + img_w_page = img.shape[1] + img = img / float(255.0) + img = resize_image(img, img_height_model, img_width_model) + + label_p_pred = model.predict(img.reshape(1, img.shape[0], img.shape[1], img.shape[2])) + + seg = np.argmax(label_p_pred, axis=3)[0] + seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) + prediction_true = resize_image(seg_color, img_h_page, img_w_page) + prediction_true = prediction_true.astype(np.uint8) + + + else: + if img.shape[0] < img_height_model: + img = resize_image(img, img_height_model, img.shape[1]) + + if img.shape[1] < img_width_model: + img = resize_image(img, img.shape[0], img_width_model) + + self.logger.info("Image dimensions: %sx%s", img_height_model, img_width_model) + margin = int(marginal_of_patch_percent * img_height_model) + width_mid = img_width_model - 2 * margin + height_mid = img_height_model - 2 * margin + img = img / float(255.0) + img = img.astype(np.float16) + img_h = img.shape[0] + img_w = img.shape[1] + prediction_true = np.zeros((img_h, img_w, 3)) + mask_true = np.zeros((img_h, img_w)) + nxf = img_w / float(width_mid) + nyf = img_h / float(height_mid) + nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) + nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) + + for i in range(nxf): + for j in range(nyf): + if i == 0: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + else: + index_x_d = i * width_mid + index_x_u = index_x_d + img_width_model + if j == 0: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + else: + index_y_d = j * height_mid + index_y_u = index_y_d + img_height_model + if index_x_u > img_w: + index_x_u = img_w + index_x_d = img_w - img_width_model + if index_y_u > img_h: + index_y_u = img_h + index_y_d = img_h - img_height_model + + img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] + label_p_pred = model.predict(img_patch.reshape(1, img_patch.shape[0], img_patch.shape[1], img_patch.shape[2])) + seg = np.argmax(label_p_pred, axis=3)[0] + seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) + + if i == 0 and j == 0: + seg_color = seg_color[0 : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] + seg = seg[0 : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin] = seg + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg_color + elif i == nxf - 1 and j == nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - 0, :] + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - 0] + mask_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0] = seg + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - 0, :] = seg_color + elif i == 0 and j == nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - 0, 0 : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - 0, 0 : seg.shape[1] - margin] + mask_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + 0 : index_x_u - margin, :] = seg_color + elif i == nxf - 1 and j == 0: + seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - 0] + mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0] = seg + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg_color + elif i == 0 and j != 0 and j != nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - margin, 0 : seg.shape[1] - margin] + mask_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + 0 : index_x_u - margin, :] = seg_color + elif i == nxf - 1 and j != 0 and j != nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - 0] + mask_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - 0, :] = seg_color + elif i != 0 and i != nxf - 1 and j == 0: + seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] + seg = seg[0 : seg.shape[0] - margin, margin : seg.shape[1] - margin] + mask_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin] = seg + prediction_true[index_y_d + 0 : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg_color + elif i != 0 and i != nxf - 1 and j == nyf - 1: + seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - 0, margin : seg.shape[1] - margin] + mask_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - 0, index_x_d + margin : index_x_u - margin, :] = seg_color + else: + seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] + seg = seg[margin : seg.shape[0] - margin, margin : seg.shape[1] - margin] + mask_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin] = seg + prediction_true[index_y_d + margin : index_y_u - margin, index_x_d + margin : index_x_u - margin, :] = seg_color + + prediction_true = prediction_true.astype(np.uint8) + del model + gc.collect() + return prediction_true + + def early_page_for_num_of_column_classification(self,img_bin): + self.logger.debug("enter early_page_for_num_of_column_classification") + if self.input_binary: + img =np.copy(img_bin) + img = img.astype(np.uint8) + else: + img = self.imread() + model_page, session_page = self.start_new_session_and_model(self.model_page_dir) + img = cv2.GaussianBlur(img, (5, 5), 0) + + img_page_prediction = self.do_prediction(False, img, model_page) + + imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + thresh = cv2.dilate(thresh, KERNEL, iterations=3) + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + if len(contours)>0: + cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) + cnt = contours[np.argmax(cnt_size)] + x, y, w, h = cv2.boundingRect(cnt) + box = [x, y, w, h] + else: + box = [0, 0, img.shape[1], img.shape[0]] + croped_page, page_coord = crop_image_inside_box(box, img) + session_page.close() + del model_page + del session_page + gc.collect() + K.clear_session() + self.logger.debug("exit early_page_for_num_of_column_classification") + return croped_page, page_coord + + def extract_page(self): + self.logger.debug("enter extract_page") + cont_page = [] + model_page, session_page = self.start_new_session_and_model(self.model_page_dir) + img = cv2.GaussianBlur(self.image, (5, 5), 0) + img_page_prediction = self.do_prediction(False, img, model_page) + imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) + _, thresh = cv2.threshold(imgray, 0, 255, 0) + thresh = cv2.dilate(thresh, KERNEL, iterations=3) + contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + if len(contours)>0: + cnt_size = np.array([cv2.contourArea(contours[j]) for j in range(len(contours))]) + cnt = contours[np.argmax(cnt_size)] + x, y, w, h = cv2.boundingRect(cnt) + if x <= 30: + w += x + x = 0 + if (self.image.shape[1] - (x + w)) <= 30: + w = w + (self.image.shape[1] - (x + w)) + if y <= 30: + h = h + y + y = 0 + if (self.image.shape[0] - (y + h)) <= 30: + h = h + (self.image.shape[0] - (y + h)) + + box = [x, y, w, h] + else: + box = [0, 0, img.shape[1], img.shape[0]] + croped_page, page_coord = crop_image_inside_box(box, self.image) + cont_page.append(np.array([[page_coord[2], page_coord[0]], [page_coord[3], page_coord[0]], [page_coord[3], page_coord[1]], [page_coord[2], page_coord[1]]])) + session_page.close() + del model_page + del session_page + gc.collect() + K.clear_session() + self.logger.debug("exit extract_page") + return croped_page, page_coord, cont_page + + def extract_text_regions(self, img, patches, cols): + self.logger.debug("enter extract_text_regions") + img_height_h = img.shape[0] + img_width_h = img.shape[1] + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_fully if patches else self.model_region_dir_fully_np) + + if not patches: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + prediction_regions2 = None + else: + if cols == 1: + img2 = otsu_copy_binary(img) + img2 = img2.astype(np.uint8) + img2 = resize_image(img2, int(img_height_h * 0.7), int(img_width_h * 0.7)) + marginal_of_patch_percent = 0.1 + prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent) + prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) + + if cols == 2: + img2 = otsu_copy_binary(img) + img2 = img2.astype(np.uint8) + img2 = resize_image(img2, int(img_height_h * 0.4), int(img_width_h * 0.4)) + marginal_of_patch_percent = 0.1 + prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent) + prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) + + elif cols > 2: + img2 = otsu_copy_binary(img) + img2 = img2.astype(np.uint8) + img2 = resize_image(img2, int(img_height_h * 0.3), int(img_width_h * 0.3)) + marginal_of_patch_percent = 0.1 + prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent) + prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) + + if cols == 2: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + if img_width_h >= 2000: + img = resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)) + img = img.astype(np.uint8) + + if cols == 1: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img = resize_image(img, int(img_height_h * 0.5), int(img_width_h * 0.5)) + img = img.astype(np.uint8) + + if cols == 3: + if (self.scale_x == 1 and img_width_h > 3000) or (self.scale_x != 1 and img_width_h > 2800): + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img = resize_image(img, int(img_height_h * 2800 / float(img_width_h)), 2800) + else: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + + if cols == 4: + if (self.scale_x == 1 and img_width_h > 4000) or (self.scale_x != 1 and img_width_h > 3700): + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img= resize_image(img, int(img_height_h * 3700 / float(img_width_h)), 3700) + else: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img= resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)) + + if cols == 5: + if self.scale_x == 1 and img_width_h > 5000: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img= resize_image(img, int(img_height_h * 0.7), int(img_width_h * 0.7)) + else: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img= resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9) ) + + if cols >= 6: + if img_width_h > 5600: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img= resize_image(img, int(img_height_h * 5600 / float(img_width_h)), 5600) + else: + img = otsu_copy_binary(img) + img = img.astype(np.uint8) + img= resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)) + + marginal_of_patch_percent = 0.1 + prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent) + prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) + + session_region.close() + del model_region + del session_region + gc.collect() + + self.logger.debug("exit extract_text_regions") + return prediction_regions, prediction_regions2 + + def get_slopes_and_deskew_new(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): + self.logger.debug("enter get_slopes_and_deskew_new") + num_cores = cpu_count() + queue_of_all_params = Queue() + + processes = [] + nh = np.linspace(0, len(boxes), num_cores + 1) + indexes_by_text_con = np.array(range(len(contours_par))) + for i in range(num_cores): + boxes_per_process = boxes[int(nh[i]) : int(nh[i + 1])] + contours_per_process = contours[int(nh[i]) : int(nh[i + 1])] + contours_par_per_process = contours_par[int(nh[i]) : int(nh[i + 1])] + indexes_text_con_per_process = indexes_by_text_con[int(nh[i]) : int(nh[i + 1])] + + processes.append(Process(target=self.do_work_of_slopes_new, args=(queue_of_all_params, boxes_per_process, textline_mask_tot, contours_per_process, contours_par_per_process, indexes_text_con_per_process, image_page_rotated, slope_deskew))) + for i in range(num_cores): + processes[i].start() + + slopes = [] + all_found_texline_polygons = [] + all_found_text_regions = [] + all_found_text_regions_par = [] + boxes = [] + all_box_coord = [] + all_index_text_con = [] + for i in range(num_cores): + list_all_par = queue_of_all_params.get(True) + slopes_for_sub_process = list_all_par[0] + polys_for_sub_process = list_all_par[1] + boxes_for_sub_process = list_all_par[2] + contours_for_subprocess = list_all_par[3] + contours_par_for_subprocess = list_all_par[4] + boxes_coord_for_subprocess = list_all_par[5] + indexes_for_subprocess = list_all_par[6] + for j in range(len(slopes_for_sub_process)): + slopes.append(slopes_for_sub_process[j]) + all_found_texline_polygons.append(polys_for_sub_process[j]) + boxes.append(boxes_for_sub_process[j]) + all_found_text_regions.append(contours_for_subprocess[j]) + all_found_text_regions_par.append(contours_par_for_subprocess[j]) + all_box_coord.append(boxes_coord_for_subprocess[j]) + all_index_text_con.append(indexes_for_subprocess[j]) + for i in range(num_cores): + processes[i].join() + self.logger.debug('slopes %s', slopes) + self.logger.debug("exit get_slopes_and_deskew_new") + return slopes, all_found_texline_polygons, boxes, all_found_text_regions, all_found_text_regions_par, all_box_coord, all_index_text_con + + def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, mask_texts_only, num_col, scale_par, slope_deskew): + self.logger.debug("enter get_slopes_and_deskew_new_curved") + num_cores = cpu_count() + queue_of_all_params = Queue() + + processes = [] + nh = np.linspace(0, len(boxes), num_cores + 1) + indexes_by_text_con = np.array(range(len(contours_par))) + + for i in range(num_cores): + boxes_per_process = boxes[int(nh[i]) : int(nh[i + 1])] + contours_per_process = contours[int(nh[i]) : int(nh[i + 1])] + contours_par_per_process = contours_par[int(nh[i]) : int(nh[i + 1])] + indexes_text_con_per_process = indexes_by_text_con[int(nh[i]) : int(nh[i + 1])] + + processes.append(Process(target=self.do_work_of_slopes_new_curved, args=(queue_of_all_params, boxes_per_process, textline_mask_tot, contours_per_process, contours_par_per_process, image_page_rotated, mask_texts_only, num_col, scale_par, indexes_text_con_per_process, slope_deskew))) + + for i in range(num_cores): + processes[i].start() + + slopes = [] + all_found_texline_polygons = [] + all_found_text_regions = [] + all_found_text_regions_par = [] + boxes = [] + all_box_coord = [] + all_index_text_con = [] + + for i in range(num_cores): + list_all_par = queue_of_all_params.get(True) + polys_for_sub_process = list_all_par[0] + boxes_for_sub_process = list_all_par[1] + contours_for_subprocess = list_all_par[2] + contours_par_for_subprocess = list_all_par[3] + boxes_coord_for_subprocess = list_all_par[4] + indexes_for_subprocess = list_all_par[5] + slopes_for_sub_process = list_all_par[6] + for j in range(len(polys_for_sub_process)): + slopes.append(slopes_for_sub_process[j]) + all_found_texline_polygons.append(polys_for_sub_process[j][::-1]) + boxes.append(boxes_for_sub_process[j]) + all_found_text_regions.append(contours_for_subprocess[j]) + all_found_text_regions_par.append(contours_par_for_subprocess[j]) + all_box_coord.append(boxes_coord_for_subprocess[j]) + all_index_text_con.append(indexes_for_subprocess[j]) + + for i in range(num_cores): + processes[i].join() + # print(slopes,'slopes') + return all_found_texline_polygons, boxes, all_found_text_regions, all_found_text_regions_par, all_box_coord, all_index_text_con, slopes + + def do_work_of_slopes_new_curved(self, queue_of_all_params, boxes_text, textline_mask_tot_ea, contours_per_process, contours_par_per_process, image_page_rotated, mask_texts_only, num_col, scale_par, indexes_r_con_per_pro, slope_deskew): + self.logger.debug("enter do_work_of_slopes_new_curved") + slopes_per_each_subprocess = [] + bounding_box_of_textregion_per_each_subprocess = [] + textlines_rectangles_per_each_subprocess = [] + contours_textregion_per_each_subprocess = [] + contours_textregion_par_per_each_subprocess = [] + all_box_coord_per_process = [] + index_by_text_region_contours = [] + + textline_cnt_separated = np.zeros(textline_mask_tot_ea.shape) + + for mv in range(len(boxes_text)): + + all_text_region_raw = textline_mask_tot_ea[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]] + all_text_region_raw = all_text_region_raw.astype(np.uint8) + img_int_p = all_text_region_raw[:, :] + + # img_int_p=cv2.erode(img_int_p,KERNEL,iterations = 2) + # plt.imshow(img_int_p) + # plt.show() + + if img_int_p.shape[0] / img_int_p.shape[1] < 0.1: + slopes_per_each_subprocess.append(0) + slope_for_all = [slope_deskew][0] + else: + try: + textline_con, hierarchy = return_contours_of_image(img_int_p) + textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, hierarchy, max_area=1, min_area=0.0008) + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) + if self.isNaN(y_diff_mean): + slope_for_all = MAX_SLOPE + else: + sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) + img_int_p[img_int_p > 0] = 1 + slope_for_all = return_deskew_slop(img_int_p, sigma_des, plotter=self.plotter) + + if abs(slope_for_all) < 0.5: + slope_for_all = [slope_deskew][0] + + except Exception as why: + self.logger.error(why) + slope_for_all = MAX_SLOPE + + if slope_for_all == MAX_SLOPE: + slope_for_all = [slope_deskew][0] + slopes_per_each_subprocess.append(slope_for_all) + + index_by_text_region_contours.append(indexes_r_con_per_pro[mv]) + _, crop_coor = crop_image_inside_box(boxes_text[mv], image_page_rotated) + + if abs(slope_for_all) < 45: + # all_box_coord.append(crop_coor) + textline_region_in_image = np.zeros(textline_mask_tot_ea.shape) + cnt_o_t_max = contours_par_per_process[mv] + x, y, w, h = cv2.boundingRect(cnt_o_t_max) + mask_biggest = np.zeros(mask_texts_only.shape) + mask_biggest = cv2.fillPoly(mask_biggest, pts=[cnt_o_t_max], color=(1, 1, 1)) + mask_region_in_patch_region = mask_biggest[y : y + h, x : x + w] + textline_biggest_region = mask_biggest * textline_mask_tot_ea + + # print(slope_for_all,'slope_for_all') + textline_rotated_separated = separate_lines_new2(textline_biggest_region[y : y + h, x : x + w], 0, num_col, slope_for_all, plotter=self.plotter) + + # new line added + ##print(np.shape(textline_rotated_separated),np.shape(mask_biggest)) + textline_rotated_separated[mask_region_in_patch_region[:, :] != 1] = 0 + # till here + + textline_cnt_separated[y : y + h, x : x + w] = textline_rotated_separated + textline_region_in_image[y : y + h, x : x + w] = textline_rotated_separated + + # plt.imshow(textline_region_in_image) + # plt.show() + # plt.imshow(textline_cnt_separated) + # plt.show() + + pixel_img = 1 + cnt_textlines_in_image = return_contours_of_interested_textline(textline_region_in_image, pixel_img) + + textlines_cnt_per_region = [] + for jjjj in range(len(cnt_textlines_in_image)): + mask_biggest2 = np.zeros(mask_texts_only.shape) + mask_biggest2 = cv2.fillPoly(mask_biggest2, pts=[cnt_textlines_in_image[jjjj]], color=(1, 1, 1)) + if num_col + 1 == 1: + mask_biggest2 = cv2.dilate(mask_biggest2, KERNEL, iterations=5) + else: + mask_biggest2 = cv2.dilate(mask_biggest2, KERNEL, iterations=4) + + pixel_img = 1 + mask_biggest2 = resize_image(mask_biggest2, int(mask_biggest2.shape[0] * scale_par), int(mask_biggest2.shape[1] * scale_par)) + cnt_textlines_in_image_ind = return_contours_of_interested_textline(mask_biggest2, pixel_img) + try: + textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]) + except Exception as why: + self.logger.error(why) + else: + add_boxes_coor_into_textlines = True + textlines_cnt_per_region = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contours_par_per_process[mv], boxes_text[mv], add_boxes_coor_into_textlines) + add_boxes_coor_into_textlines = False + # print(np.shape(textlines_cnt_per_region),'textlines_cnt_per_region') + + textlines_rectangles_per_each_subprocess.append(textlines_cnt_per_region) + bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv]) + contours_textregion_per_each_subprocess.append(contours_per_process[mv]) + contours_textregion_par_per_each_subprocess.append(contours_par_per_process[mv]) + all_box_coord_per_process.append(crop_coor) + + queue_of_all_params.put([textlines_rectangles_per_each_subprocess, bounding_box_of_textregion_per_each_subprocess, contours_textregion_per_each_subprocess, contours_textregion_par_per_each_subprocess, all_box_coord_per_process, index_by_text_region_contours, slopes_per_each_subprocess]) + + def do_work_of_slopes_new(self, queue_of_all_params, boxes_text, textline_mask_tot_ea, contours_per_process, contours_par_per_process, indexes_r_con_per_pro, image_page_rotated, slope_deskew): + self.logger.debug('enter do_work_of_slopes_new') + slopes_per_each_subprocess = [] + bounding_box_of_textregion_per_each_subprocess = [] + textlines_rectangles_per_each_subprocess = [] + contours_textregion_per_each_subprocess = [] + contours_textregion_par_per_each_subprocess = [] + all_box_coord_per_process = [] + index_by_text_region_contours = [] + for mv in range(len(boxes_text)): + _, crop_coor = crop_image_inside_box(boxes_text[mv],image_page_rotated) + mask_textline = np.zeros((textline_mask_tot_ea.shape)) + mask_textline = cv2.fillPoly(mask_textline,pts=[contours_per_process[mv]],color=(1,1,1)) + all_text_region_raw = (textline_mask_tot_ea*mask_textline[:,:])[boxes_text[mv][1]:boxes_text[mv][1]+boxes_text[mv][3] , boxes_text[mv][0]:boxes_text[mv][0]+boxes_text[mv][2] ] + all_text_region_raw=all_text_region_raw.astype(np.uint8) + img_int_p=all_text_region_raw[:,:]#self.all_text_region_raw[mv] + img_int_p=cv2.erode(img_int_p,KERNEL,iterations = 2) + + if img_int_p.shape[0]/img_int_p.shape[1]<0.1: + slopes_per_each_subprocess.append(0) + slope_for_all = [slope_deskew][0] + all_text_region_raw = textline_mask_tot_ea[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]] + cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contours_par_per_process[mv], boxes_text[mv], 0) + textlines_rectangles_per_each_subprocess.append(cnt_clean_rot) + index_by_text_region_contours.append(indexes_r_con_per_pro[mv]) + bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv]) + else: + try: + textline_con, hierarchy = return_contours_of_image(img_int_p) + textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, hierarchy, max_area=1, min_area=0.00008) + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) + if self.isNaN(y_diff_mean): + slope_for_all = MAX_SLOPE + else: + sigma_des = int(y_diff_mean * (4.0 / 40.0)) + if sigma_des < 1: + sigma_des = 1 + img_int_p[img_int_p > 0] = 1 + slope_for_all = return_deskew_slop(img_int_p, sigma_des, plotter=self.plotter) + if abs(slope_for_all) <= 0.5: + slope_for_all = [slope_deskew][0] + except Exception as why: + self.logger.error(why) + slope_for_all = MAX_SLOPE + if slope_for_all == MAX_SLOPE: + slope_for_all = [slope_deskew][0] + slopes_per_each_subprocess.append(slope_for_all) + mask_only_con_region = np.zeros(textline_mask_tot_ea.shape) + mask_only_con_region = cv2.fillPoly(mask_only_con_region, pts=[contours_par_per_process[mv]], color=(1, 1, 1)) + + # plt.imshow(mask_only_con_region) + # plt.show() + all_text_region_raw = np.copy(textline_mask_tot_ea[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]]) + mask_only_con_region = mask_only_con_region[boxes_text[mv][1] : boxes_text[mv][1] + boxes_text[mv][3], boxes_text[mv][0] : boxes_text[mv][0] + boxes_text[mv][2]] + + ##plt.imshow(textline_mask_tot_ea) + ##plt.show() + ##plt.imshow(all_text_region_raw) + ##plt.show() + ##plt.imshow(mask_only_con_region) + ##plt.show() + + all_text_region_raw[mask_only_con_region == 0] = 0 + cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contours_par_per_process[mv], boxes_text[mv]) + + textlines_rectangles_per_each_subprocess.append(cnt_clean_rot) + index_by_text_region_contours.append(indexes_r_con_per_pro[mv]) + bounding_box_of_textregion_per_each_subprocess.append(boxes_text[mv]) + + contours_textregion_per_each_subprocess.append(contours_per_process[mv]) + contours_textregion_par_per_each_subprocess.append(contours_par_per_process[mv]) + all_box_coord_per_process.append(crop_coor) + queue_of_all_params.put([slopes_per_each_subprocess, textlines_rectangles_per_each_subprocess, bounding_box_of_textregion_per_each_subprocess, contours_textregion_per_each_subprocess, contours_textregion_par_per_each_subprocess, all_box_coord_per_process, index_by_text_region_contours]) + + def textline_contours(self, img, patches, scaler_h, scaler_w): + self.logger.debug('enter textline_contours') + + model_textline, session_textline = self.start_new_session_and_model(self.model_textline_dir if patches else self.model_textline_dir_np) + img = img.astype(np.uint8) + img_org = np.copy(img) + img_h = img_org.shape[0] + img_w = img_org.shape[1] + img = resize_image(img_org, int(img_org.shape[0] * scaler_h), int(img_org.shape[1] * scaler_w)) + prediction_textline = self.do_prediction(patches, img, model_textline) + prediction_textline = resize_image(prediction_textline, img_h, img_w) + prediction_textline_longshot = self.do_prediction(False, img, model_textline) + prediction_textline_longshot_true_size = resize_image(prediction_textline_longshot, img_h, img_w) + + session_textline.close() + + + return prediction_textline[:, :, 0], prediction_textline_longshot_true_size[:, :, 0] + + def do_work_of_slopes(self, q, poly, box_sub, boxes_per_process, textline_mask_tot, contours_per_process): + self.logger.debug('enter do_work_of_slopes') + slope_biggest = 0 + slopes_sub = [] + boxes_sub_new = [] + poly_sub = [] + for mv in range(len(boxes_per_process)): + crop_img, _ = crop_image_inside_box(boxes_per_process[mv], np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) + crop_img = crop_img[:, :, 0] + crop_img = cv2.erode(crop_img, KERNEL, iterations=2) + try: + textline_con, hierarchy = return_contours_of_image(crop_img) + textline_con_fil = filter_contours_area_of_image(crop_img, textline_con, hierarchy, max_area=1, min_area=0.0008) + y_diff_mean = find_contours_mean_y_diff(textline_con_fil) + sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) + crop_img[crop_img > 0] = 1 + slope_corresponding_textregion = return_deskew_slop(crop_img, sigma_des, plotter=self.plotter) + except Exception as why: + self.logger.error(why) + slope_corresponding_textregion = MAX_SLOPE + + if slope_corresponding_textregion == MAX_SLOPE: + slope_corresponding_textregion = slope_biggest + slopes_sub.append(slope_corresponding_textregion) + + cnt_clean_rot = textline_contours_postprocessing(crop_img, slope_corresponding_textregion, contours_per_process[mv], boxes_per_process[mv]) + + poly_sub.append(cnt_clean_rot) + boxes_sub_new.append(boxes_per_process[mv]) + + q.put(slopes_sub) + poly.put(poly_sub) + box_sub.put(boxes_sub_new) + + def get_regions_from_xy_2models(self,img,is_image_enhanced, num_col_classifier): + self.logger.debug("enter get_regions_from_xy_2models") + erosion_hurts = False + img_org = np.copy(img) + img_height_h = img_org.shape[0] + img_width_h = img_org.shape[1] + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) + + ratio_y=1.3 + ratio_x=1 + + img = resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) + + prediction_regions_org_y = self.do_prediction(True, img, model_region) + prediction_regions_org_y = resize_image(prediction_regions_org_y, img_height_h, img_width_h ) + + #plt.imshow(prediction_regions_org_y[:,:,0]) + #plt.show() + prediction_regions_org_y = prediction_regions_org_y[:,:,0] + mask_zeros_y = (prediction_regions_org_y[:,:]==0)*1 + + ##img_only_regions_with_sep = ( (prediction_regions_org_y[:,:] != 3) & (prediction_regions_org_y[:,:] != 0) )*1 + img_only_regions_with_sep = ( prediction_regions_org_y[:,:] == 1 )*1 + img_only_regions_with_sep = img_only_regions_with_sep.astype(np.uint8) + + try: + img_only_regions = cv2.erode(img_only_regions_with_sep[:,:], KERNEL, iterations=20) + + _, _ = find_num_col(img_only_regions, multiplier=6.0) + + img = resize_image(img_org, int(img_org.shape[0]), int(img_org.shape[1]*(1.2 if is_image_enhanced else 1))) + + prediction_regions_org = self.do_prediction(True, img, model_region) + prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) + + ##plt.imshow(prediction_regions_org[:,:,0]) + ##plt.show() + prediction_regions_org=prediction_regions_org[:,:,0] + prediction_regions_org[(prediction_regions_org[:,:]==1) & (mask_zeros_y[:,:]==1)]=0 + + session_region.close() + del model_region + del session_region + gc.collect() + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p2) + img = resize_image(img_org, int(img_org.shape[0]), int(img_org.shape[1])) + prediction_regions_org2 = self.do_prediction(True, img, model_region, 0.2) + prediction_regions_org2=resize_image(prediction_regions_org2, img_height_h, img_width_h ) + + + session_region.close() + del model_region + del session_region + gc.collect() + + mask_zeros2 = (prediction_regions_org2[:,:,0] == 0) + mask_lines2 = (prediction_regions_org2[:,:,0] == 3) + text_sume_early = (prediction_regions_org[:,:] == 1).sum() + prediction_regions_org_copy = np.copy(prediction_regions_org) + prediction_regions_org_copy[(prediction_regions_org_copy[:,:]==1) & (mask_zeros2[:,:]==1)] = 0 + text_sume_second = ((prediction_regions_org_copy[:,:]==1)*1).sum() + + rate_two_models = text_sume_second / float(text_sume_early) * 100 + + self.logger.info("ratio_of_two_models: %s", rate_two_models) + if not(is_image_enhanced and rate_two_models < RATIO_OF_TWO_MODEL_THRESHOLD): + prediction_regions_org = np.copy(prediction_regions_org_copy) + + + + prediction_regions_org[(mask_lines2[:,:]==1) & (prediction_regions_org[:,:]==0)]=3 + mask_lines_only=(prediction_regions_org[:,:]==3)*1 + prediction_regions_org = cv2.erode(prediction_regions_org[:,:], KERNEL, iterations=2) + + #plt.imshow(text_region2_1st_channel) + #plt.show() + + prediction_regions_org = cv2.dilate(prediction_regions_org[:,:], KERNEL, iterations=2) + + + if rate_two_models<=40: + if self.input_binary: + prediction_bin = np.copy(img_org) + else: + model_bin, session_bin = self.start_new_session_and_model(self.model_dir_of_binarization) + prediction_bin = self.do_prediction(True, img_org, model_bin) + prediction_bin = resize_image(prediction_bin, img_height_h, img_width_h ) + + prediction_bin=prediction_bin[:,:,0] + prediction_bin = (prediction_bin[:,:]==0)*1 + prediction_bin = prediction_bin*255 + + prediction_bin =np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) + + session_bin.close() + del model_bin + del session_bin + gc.collect() + + + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) + ratio_y=1 + ratio_x=1 + + + img = resize_image(prediction_bin, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) + + prediction_regions_org = self.do_prediction(True, img, model_region) + prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) + prediction_regions_org=prediction_regions_org[:,:,0] + + mask_lines_only=(prediction_regions_org[:,:]==3)*1 + session_region.close() + del model_region + del session_region + gc.collect() + + + mask_texts_only=(prediction_regions_org[:,:]==1)*1 + mask_images_only=(prediction_regions_org[:,:]==2)*1 + + + + polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) + polygons_lines_xml = textline_con_fil = filter_contours_area_of_image(mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) + + polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only, 1, 0.00001) + polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only, 1, 0.00001) + + text_regions_p_true = np.zeros(prediction_regions_org.shape) + text_regions_p_true = cv2.fillPoly(text_regions_p_true,pts = polygons_of_only_lines, color=(3, 3, 3)) + text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 + + text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) + + + + K.clear_session() + return text_regions_p_true, erosion_hurts, polygons_lines_xml + except: + + if self.input_binary: + prediction_bin = np.copy(img_org) + else: + session_region.close() + del model_region + del session_region + gc.collect() + + model_bin, session_bin = self.start_new_session_and_model(self.model_dir_of_binarization) + prediction_bin = self.do_prediction(True, img_org, model_bin) + prediction_bin = resize_image(prediction_bin, img_height_h, img_width_h ) + prediction_bin=prediction_bin[:,:,0] + + prediction_bin = (prediction_bin[:,:]==0)*1 + + prediction_bin = prediction_bin*255 + + prediction_bin =np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) + + + + session_bin.close() + del model_bin + del session_bin + gc.collect() + + + + model_region, session_region = self.start_new_session_and_model(self.model_region_dir_p_ens) + ratio_y=1 + ratio_x=1 + + + img = resize_image(prediction_bin, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) + + prediction_regions_org = self.do_prediction(True, img, model_region) + prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) + prediction_regions_org=prediction_regions_org[:,:,0] + + #mask_lines_only=(prediction_regions_org[:,:]==3)*1 + session_region.close() + del model_region + del session_region + gc.collect() + + #img = resize_image(img_org, int(img_org.shape[0]*1), int(img_org.shape[1]*1)) + + #prediction_regions_org = self.do_prediction(True, img, model_region) + + #prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) + + #prediction_regions_org = prediction_regions_org[:,:,0] + + #prediction_regions_org[(prediction_regions_org[:,:] == 1) & (mask_zeros_y[:,:] == 1)]=0 + #session_region.close() + #del model_region + #del session_region + #gc.collect() + + + + + mask_lines_only = (prediction_regions_org[:,:] ==3)*1 + + mask_texts_only = (prediction_regions_org[:,:] ==1)*1 + + mask_images_only=(prediction_regions_org[:,:] ==2)*1 + + polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) + polygons_lines_xml = textline_con_fil = filter_contours_area_of_image(mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) + + + polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) + + polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) + + + text_regions_p_true = np.zeros(prediction_regions_org.shape) + + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_lines, color=(3,3,3)) + + text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 + + text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_texts, color=(1,1,1)) + + erosion_hurts = True + K.clear_session() + return text_regions_p_true, erosion_hurts, polygons_lines_xml + + def do_order_of_regions_full_layout(self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): + self.logger.debug("enter do_order_of_regions_full_layout") + cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours(contours_only_text_parent) + cx_text_only_h, cy_text_only_h, x_min_text_only_h, _, _, _, y_cor_x_min_main_h = find_new_features_of_contours(contours_only_text_parent_h) + + try: + arg_text_con = [] + for ii in range(len(cx_text_only)): + for jj in range(len(boxes)): + if (x_min_text_only[ii] + 80) >= boxes[jj][0] and (x_min_text_only[ii] + 80) < boxes[jj][1] and y_cor_x_min_main[ii] >= boxes[jj][2] and y_cor_x_min_main[ii] < boxes[jj][3]: + arg_text_con.append(jj) + break + args_contours = np.array(range(len(arg_text_con))) + arg_text_con_h = [] + for ii in range(len(cx_text_only_h)): + for jj in range(len(boxes)): + if (x_min_text_only_h[ii] + 80) >= boxes[jj][0] and (x_min_text_only_h[ii] + 80) < boxes[jj][1] and y_cor_x_min_main_h[ii] >= boxes[jj][2] and y_cor_x_min_main_h[ii] < boxes[jj][3]: + arg_text_con_h.append(jj) + break + args_contours_h = np.array(range(len(arg_text_con_h))) + + order_by_con_head = np.zeros(len(arg_text_con_h)) + order_by_con_main = np.zeros(len(arg_text_con)) + + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] + for iij in range(len(boxes)): + + args_contours_box = args_contours[np.array(arg_text_con) == iij] + args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] + con_inter_box = [] + con_inter_box_h = [] + + for box in args_contours_box: + con_inter_box.append(contours_only_text_parent[box]) + + for box in args_contours_box_h: + con_inter_box_h.append(contours_only_text_parent_h[box]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + + for zahler, _ in enumerate(args_contours_box): + arg_order_v = indexes_sorted_main[zahler] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + + for zahler, _ in enumerate(args_contours_box_h): + arg_order_v = indexes_sorted_head[zahler] + order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + + for jji in range(len(id_of_texts)): + order_of_texts_tot.append(order_of_texts[jji] + ref_point) + id_of_texts_tot.append(id_of_texts[jji]) + ref_point += len(id_of_texts) + + order_of_texts_tot = [] + for tj1 in range(len(contours_only_text_parent)): + order_of_texts_tot.append(int(order_by_con_main[tj1])) + + for tj1 in range(len(contours_only_text_parent_h)): + order_of_texts_tot.append(int(order_by_con_head[tj1])) + + order_text_new = [] + for iii in range(len(order_of_texts_tot)): + order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + + except Exception as why: + self.logger.error(why) + arg_text_con = [] + for ii in range(len(cx_text_only)): + for jj in range(len(boxes)): + if cx_text_only[ii] >= boxes[jj][0] and cx_text_only[ii] < boxes[jj][1] and cy_text_only[ii] >= boxes[jj][2] and cy_text_only[ii] < boxes[jj][3]: # this is valid if the center of region identify in which box it is located + arg_text_con.append(jj) + break + args_contours = np.array(range(len(arg_text_con))) + + order_by_con_main = np.zeros(len(arg_text_con)) + + ############################# head + + arg_text_con_h = [] + for ii in range(len(cx_text_only_h)): + for jj in range(len(boxes)): + if cx_text_only_h[ii] >= boxes[jj][0] and cx_text_only_h[ii] < boxes[jj][1] and cy_text_only_h[ii] >= boxes[jj][2] and cy_text_only_h[ii] < boxes[jj][3]: # this is valid if the center of region identify in which box it is located + arg_text_con_h.append(jj) + break + args_contours_h = np.array(range(len(arg_text_con_h))) + + order_by_con_head = np.zeros(len(arg_text_con_h)) + + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] + for iij, _ in enumerate(boxes): + args_contours_box = args_contours[np.array(arg_text_con) == iij] + args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] + con_inter_box = [] + con_inter_box_h = [] + + for box in args_contours_box: + con_inter_box.append(contours_only_text_parent[box]) + + for box in args_contours_box_h: + con_inter_box_h.append(contours_only_text_parent_h[box]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] + indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] + + for zahler, _ in enumerate(args_contours_box): + arg_order_v = indexes_sorted_main[zahler] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + + for zahler, _ in enumerate(args_contours_box_h): + arg_order_v = indexes_sorted_head[zahler] + order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + + for jji, _ in enumerate(id_of_texts): + order_of_texts_tot.append(order_of_texts[jji] + ref_point) + id_of_texts_tot.append(id_of_texts[jji]) + ref_point += len(id_of_texts) + + order_of_texts_tot = [] + for tj1 in range(len(contours_only_text_parent)): + order_of_texts_tot.append(int(order_by_con_main[tj1])) + + for tj1 in range(len(contours_only_text_parent_h)): + order_of_texts_tot.append(int(order_by_con_head[tj1])) + + order_text_new = [] + for iii in range(len(order_of_texts_tot)): + order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + return order_text_new, id_of_texts_tot + + def do_order_of_regions_no_full_layout(self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): + self.logger.debug("enter do_order_of_regions_no_full_layout") + cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours(contours_only_text_parent) + + try: + arg_text_con = [] + for ii in range(len(cx_text_only)): + for jj in range(len(boxes)): + if (x_min_text_only[ii] + 80) >= boxes[jj][0] and (x_min_text_only[ii] + 80) < boxes[jj][1] and y_cor_x_min_main[ii] >= boxes[jj][2] and y_cor_x_min_main[ii] < boxes[jj][3]: + arg_text_con.append(jj) + break + args_contours = np.array(range(len(arg_text_con))) + order_by_con_main = np.zeros(len(arg_text_con)) + + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] + for iij in range(len(boxes)): + args_contours_box = args_contours[np.array(arg_text_con) == iij] + con_inter_box = [] + con_inter_box_h = [] + for i in range(len(args_contours_box)): + con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + + for zahler, _ in enumerate(args_contours_box): + arg_order_v = indexes_sorted_main[zahler] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + + for jji, _ in enumerate(id_of_texts): + order_of_texts_tot.append(order_of_texts[jji] + ref_point) + id_of_texts_tot.append(id_of_texts[jji]) + ref_point += len(id_of_texts) + + order_of_texts_tot = [] + for tj1 in range(len(contours_only_text_parent)): + order_of_texts_tot.append(int(order_by_con_main[tj1])) + + order_text_new = [] + for iii in range(len(order_of_texts_tot)): + order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + + except Exception as why: + self.logger.error(why) + arg_text_con = [] + for ii in range(len(cx_text_only)): + for jj in range(len(boxes)): + if cx_text_only[ii] >= boxes[jj][0] and cx_text_only[ii] < boxes[jj][1] and cy_text_only[ii] >= boxes[jj][2] and cy_text_only[ii] < boxes[jj][3]: # this is valid if the center of region identify in which box it is located + arg_text_con.append(jj) + break + args_contours = np.array(range(len(arg_text_con))) + + order_by_con_main = np.zeros(len(arg_text_con)) + + ref_point = 0 + order_of_texts_tot = [] + id_of_texts_tot = [] + for iij in range(len(boxes)): + args_contours_box = args_contours[np.array(arg_text_con) == iij] + con_inter_box = [] + con_inter_box_h = [] + + for i in range(len(args_contours_box)): + con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) + + indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions(textline_mask_tot[int(boxes[iij][2]) : int(boxes[iij][3]), int(boxes[iij][0]) : int(boxes[iij][1])], con_inter_box, con_inter_box_h, boxes[iij][2]) + + order_of_texts, id_of_texts = order_and_id_of_texts(con_inter_box, con_inter_box_h, matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) + + indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] + indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] + + for zahler, _ in enumerate(args_contours_box): + arg_order_v = indexes_sorted_main[zahler] + order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = np.where(indexes_sorted == arg_order_v)[0][0] + ref_point + + for jji, _ in enumerate(id_of_texts): + order_of_texts_tot.append(order_of_texts[jji] + ref_point) + id_of_texts_tot.append(id_of_texts[jji]) + ref_point += len(id_of_texts) + + order_of_texts_tot = [] + for tj1 in range(len(contours_only_text_parent)): + order_of_texts_tot.append(int(order_by_con_main[tj1])) + + order_text_new = [] + for iii in range(len(order_of_texts_tot)): + order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) + + return order_text_new, id_of_texts_tot + + def do_order_of_regions(self, *args, **kwargs): + if self.full_layout: + return self.do_order_of_regions_full_layout(*args, **kwargs) + return self.do_order_of_regions_no_full_layout(*args, **kwargs) + + def run_graphics_and_columns(self, text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts): + img_g = self.imread(grayscale=True, uint8=True) + + img_g3 = np.zeros((img_g.shape[0], img_g.shape[1], 3)) + img_g3 = img_g3.astype(np.uint8) + img_g3[:, :, 0] = img_g[:, :] + img_g3[:, :, 1] = img_g[:, :] + img_g3[:, :, 2] = img_g[:, :] + + image_page, page_coord, cont_page = self.extract_page() + if self.plotter: + self.plotter.save_page_image(image_page) + + text_regions_p_1 = text_regions_p_1[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] + mask_images = (text_regions_p_1[:, :] == 2) * 1 + mask_images = mask_images.astype(np.uint8) + mask_images = cv2.erode(mask_images[:, :], KERNEL, iterations=10) + mask_lines = (text_regions_p_1[:, :] == 3) * 1 + mask_lines = mask_lines.astype(np.uint8) + img_only_regions_with_sep = ((text_regions_p_1[:, :] != 3) & (text_regions_p_1[:, :] != 0)) * 1 + img_only_regions_with_sep = img_only_regions_with_sep.astype(np.uint8) + + + if erosion_hurts: + img_only_regions = np.copy(img_only_regions_with_sep[:,:]) + else: + img_only_regions = cv2.erode(img_only_regions_with_sep[:,:], KERNEL, iterations=6) + + + try: + num_col, _ = find_num_col(img_only_regions, multiplier=6.0) + num_col = num_col + 1 + if not num_column_is_classified: + num_col_classifier = num_col + 1 + except Exception as why: + self.logger.error(why) + num_col = None + return num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, text_regions_p_1, cont_page + + def run_enhancement(self): + self.logger.info("resize and enhance image") + is_image_enhanced, img_org, img_res, num_col_classifier, num_column_is_classified, img_bin = self.resize_and_enhance_image_with_column_classifier() + self.logger.info("Image is %senhanced", '' if is_image_enhanced else 'not ') + K.clear_session() + scale = 1 + if is_image_enhanced: + if self.allow_enhancement: + img_res = img_res.astype(np.uint8) + self.get_image_and_scales(img_org, img_res, scale) + else: + self.get_image_and_scales_after_enhancing(img_org, img_res) + else: + if self.allow_enhancement: + self.get_image_and_scales(img_org, img_res, scale) + else: + self.get_image_and_scales(img_org, img_res, scale) + if self.allow_scaling: + img_org, img_res, is_image_enhanced = self.resize_image_with_column_classifier(is_image_enhanced, img_bin) + self.get_image_and_scales_after_enhancing(img_org, img_res) + return img_res, is_image_enhanced, num_col_classifier, num_column_is_classified + + def run_textline(self, image_page): + scaler_h_textline = 1 # 1.2#1.2 + scaler_w_textline = 1 # 0.9#1 + textline_mask_tot_ea, _ = self.textline_contours(image_page, True, scaler_h_textline, scaler_w_textline) + K.clear_session() + if self.plotter: + self.plotter.save_plot_of_textlines(textline_mask_tot_ea, image_page) + return textline_mask_tot_ea + + def run_deskew(self, textline_mask_tot_ea): + sigma = 2 + main_page_deskew = True + slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), sigma, main_page_deskew, plotter=self.plotter) + slope_first = 0 + + if self.plotter: + self.plotter.save_deskewed_image(slope_deskew) + self.logger.info("slope_deskew: %s", slope_deskew) + return slope_deskew, slope_first + + def run_marginals(self, image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1): + image_page_rotated, textline_mask_tot = image_page[:, :], textline_mask_tot_ea[:, :] + textline_mask_tot[mask_images[:, :] == 1] = 0 + + text_regions_p_1[mask_lines[:, :] == 1] = 3 + text_regions_p = text_regions_p_1[:, :] + text_regions_p = np.array(text_regions_p) + + if num_col_classifier in (1, 2): + try: + regions_without_separators = (text_regions_p[:, :] == 1) * 1 + regions_without_separators = regions_without_separators.astype(np.uint8) + text_regions_p = get_marginals(rotate_image(regions_without_separators, slope_deskew), text_regions_p, num_col_classifier, slope_deskew, kernel=KERNEL) + except Exception as e: + self.logger.error("exception %s", e) + + if self.plotter: + self.plotter.save_plot_of_layout_main_all(text_regions_p, image_page) + self.plotter.save_plot_of_layout_main(text_regions_p, image_page) + return textline_mask_tot, text_regions_p, image_page_rotated + + def run_boxes_no_full_layout(self, image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, erosion_hurts): + self.logger.debug('enter run_boxes_no_full_layout') + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + _, textline_mask_tot_d, text_regions_p_1_n = rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, slope_deskew) + text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1]) + textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1]) + regions_without_separators_d = (text_regions_p_1_n[:, :] == 1) * 1 + regions_without_separators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions) + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + text_regions_p_1_n = None + textline_mask_tot_d = None + regions_without_separators_d = None + pixel_lines = 3 + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + _, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) + K.clear_session() + + self.logger.info("num_col_classifier: %s", num_col_classifier) + + if num_col_classifier >= 3: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + regions_without_separators = regions_without_separators.astype(np.uint8) + regions_without_separators = cv2.erode(regions_without_separators[:, :], KERNEL, iterations=6) + else: + regions_without_separators_d = regions_without_separators_d.astype(np.uint8) + regions_without_separators_d = cv2.erode(regions_without_separators_d[:, :], KERNEL, iterations=6) + t1 = time.time() + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + boxes = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts) + boxes_d = None + self.logger.debug("len(boxes): %s", len(boxes)) + else: + boxes_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts) + boxes = None + self.logger.debug("len(boxes): %s", len(boxes_d)) + + self.logger.info("detecting boxes took %ss", str(time.time() - t1)) + img_revised_tab = text_regions_p[:, :] + polygons_of_images = return_contours_of_interested_region(img_revised_tab, 2) + + # plt.imshow(img_revised_tab) + # plt.show() + K.clear_session() + self.logger.debug('exit run_boxes_no_full_layout') + return polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, boxes, boxes_d + + def run_boxes_full_layout(self, image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions): + self.logger.debug('enter run_boxes_full_layout') + # set first model with second model + text_regions_p[:, :][text_regions_p[:, :] == 2] = 5 + text_regions_p[:, :][text_regions_p[:, :] == 3] = 6 + text_regions_p[:, :][text_regions_p[:, :] == 4] = 8 + + K.clear_session() + image_page = image_page.astype(np.uint8) + + regions_fully, regions_fully_only_drop = self.extract_text_regions(image_page, True, cols=num_col_classifier) + text_regions_p[:,:][regions_fully[:,:,0]==6]=6 + regions_fully_only_drop = put_drop_out_from_only_drop_model(regions_fully_only_drop, text_regions_p) + regions_fully[:, :, 0][regions_fully_only_drop[:, :, 0] == 4] = 4 + K.clear_session() + + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + regions_fully = putt_bb_of_drop_capitals_of_model_in_patches_in_layout(regions_fully) + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + K.clear_session() + regions_fully_np, _ = self.extract_text_regions(image_page, False, cols=num_col_classifier) + # plt.imshow(regions_fully_np[:,:,0]) + # plt.show() + if num_col_classifier > 2: + regions_fully_np[:, :, 0][regions_fully_np[:, :, 0] == 4] = 0 + else: + regions_fully_np = filter_small_drop_capitals_from_no_patch_layout(regions_fully_np, text_regions_p) + + # plt.imshow(regions_fully_np[:,:,0]) + # plt.show() + K.clear_session() + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + regions_fully = boosting_headers_by_longshot_region_segmentation(regions_fully, regions_fully_np, img_only_regions) + # plt.imshow(regions_fully[:,:,0]) + # plt.show() + text_regions_p[:, :][regions_fully[:, :, 0] == 4] = 4 + text_regions_p[:, :][regions_fully_np[:, :, 0] == 4] = 4 + #plt.imshow(text_regions_p) + #plt.show() + + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + _, textline_mask_tot_d, text_regions_p_1_n, regions_fully_n = rotation_not_90_func_full_layout(image_page, textline_mask_tot, text_regions_p, regions_fully, slope_deskew) + + text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1]) + textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1]) + regions_fully_n = resize_image(regions_fully_n, text_regions_p.shape[0], text_regions_p.shape[1]) + regions_without_separators_d = (text_regions_p_1_n[:, :] == 1) * 1 + else: + text_regions_p_1_n = None + textline_mask_tot_d = None + regions_without_separators_d = None + + regions_without_separators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions) + + K.clear_session() + img_revised_tab = np.copy(text_regions_p[:, :]) + polygons_of_images = return_contours_of_interested_region(img_revised_tab, 5) + self.logger.debug('exit run_boxes_full_layout') + return polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, regions_fully, regions_without_separators + + def run(self): + """ + Get image and scales, then extract the page of scanned image + """ + self.logger.debug("enter run") + + t0 = time.time() + img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement() + + self.logger.info("Enhancing took %ss ", str(time.time() - t0)) + + t1 = time.time() + text_regions_p_1 ,erosion_hurts, polygons_lines_xml = self.get_regions_from_xy_2models(img_res, is_image_enhanced, num_col_classifier) + self.logger.info("Textregion detection took %ss ", str(time.time() - t1)) + + t1 = time.time() + num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, text_regions_p_1, cont_page = \ + self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts) + self.logger.info("Graphics detection took %ss ", str(time.time() - t1)) + self.logger.info('cont_page %s', cont_page) + + if not num_col: + self.logger.info("No columns detected, outputting an empty PAGE-XML") + pcgts = self.writer.build_pagexml_no_full_layout([], page_coord, [], [], [], [], [], [], [], [], [], [], cont_page, []) + self.logger.info("Job done in %ss", str(time.time() - t1)) + return pcgts + + t1 = time.time() + textline_mask_tot_ea = self.run_textline(image_page) + self.logger.info("textline detection took %ss", str(time.time() - t1)) + + t1 = time.time() + slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) + self.logger.info("deskewing took %ss", str(time.time() - t1)) + t1 = time.time() + + textline_mask_tot, text_regions_p, image_page_rotated = self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, num_col_classifier, slope_deskew, text_regions_p_1) + self.logger.info("detection of marginals took %ss", str(time.time() - t1)) + t1 = time.time() + + if not self.full_layout: + polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, boxes, boxes_d = self.run_boxes_no_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, erosion_hurts) + + pixel_img = 4 + min_area_mar = 0.00001 + polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar) + + if self.full_layout: + polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, regions_fully, regions_without_separators = self.run_boxes_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, num_col_classifier, img_only_regions) + + text_only = ((img_revised_tab[:, :] == 1)) * 1 + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + text_only_d = ((text_regions_p_1_n[:, :] == 1)) * 1 + + min_con_area = 0.000005 + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + contours_only_text, hir_on_text = return_contours_of_image(text_only) + contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) + + if len(contours_only_text_parent) > 0: + areas_cnt_text = np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) + areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) + self.logger.info('areas_cnt_text %s', areas_cnt_text) + contours_biggest = contours_only_text_parent[np.argmax(areas_cnt_text)] + contours_only_text_parent = [contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz] > min_con_area] + areas_cnt_text_parent = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > min_con_area] + + index_con_parents = np.argsort(areas_cnt_text_parent) + contours_only_text_parent = list(np.array(contours_only_text_parent)[index_con_parents]) + areas_cnt_text_parent = list(np.array(areas_cnt_text_parent)[index_con_parents]) + + cx_bigest_big, cy_biggest_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest]) + cx_bigest, cy_biggest, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent) + + contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) + contours_only_text_parent_d = return_parent_contours(contours_only_text_d, hir_on_text_d) + + areas_cnt_text_d = np.array([cv2.contourArea(contours_only_text_parent_d[j]) for j in range(len(contours_only_text_parent_d))]) + areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) + + if len(areas_cnt_text_d)>0: + contours_biggest_d = contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] + index_con_parents_d=np.argsort(areas_cnt_text_d) + contours_only_text_parent_d=list(np.array(contours_only_text_parent_d)[index_con_parents_d] ) + areas_cnt_text_d=list(np.array(areas_cnt_text_d)[index_con_parents_d] ) + + cx_bigest_d_big, cy_biggest_d_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest_d]) + cx_bigest_d, cy_biggest_d, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent_d) + try: + if len(cx_bigest_d) >= 5: + cx_bigest_d_last5 = cx_bigest_d[-5:] + cy_biggest_d_last5 = cy_biggest_d[-5:] + dists_d = [math.sqrt((cx_bigest_big[0] - cx_bigest_d_last5[j]) ** 2 + (cy_biggest_big[0] - cy_biggest_d_last5[j]) ** 2) for j in range(len(cy_biggest_d_last5))] + ind_largest = len(cx_bigest_d) -5 + np.argmin(dists_d) + else: + cx_bigest_d_last5 = cx_bigest_d[-len(cx_bigest_d):] + cy_biggest_d_last5 = cy_biggest_d[-len(cx_bigest_d):] + dists_d = [math.sqrt((cx_bigest_big[0]-cx_bigest_d_last5[j])**2 + (cy_biggest_big[0]-cy_biggest_d_last5[j])**2) for j in range(len(cy_biggest_d_last5))] + ind_largest = len(cx_bigest_d) - len(cx_bigest_d) + np.argmin(dists_d) + + cx_bigest_d_big[0] = cx_bigest_d[ind_largest] + cy_biggest_d_big[0] = cy_biggest_d[ind_largest] + except Exception as why: + self.logger.error(why) + + (h, w) = text_only.shape[:2] + center = (w // 2.0, h // 2.0) + M = cv2.getRotationMatrix2D(center, slope_deskew, 1.0) + M_22 = np.array(M)[:2, :2] + p_big = np.dot(M_22, [cx_bigest_big, cy_biggest_big]) + x_diff = p_big[0] - cx_bigest_d_big + y_diff = p_big[1] - cy_biggest_d_big + + contours_only_text_parent_d_ordered = [] + for i in range(len(contours_only_text_parent)): + p = np.dot(M_22, [cx_bigest[i], cy_biggest[i]]) + p[0] = p[0] - x_diff[0] + p[1] = p[1] - y_diff[0] + dists = [math.sqrt((p[0] - cx_bigest_d[j]) ** 2 + (p[1] - cy_biggest_d[j]) ** 2) for j in range(len(cx_bigest_d))] + contours_only_text_parent_d_ordered.append(contours_only_text_parent_d[np.argmin(dists)]) + # img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) + # img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d[np.argmin(dists)]] ,color=(1,1,1)) + # plt.imshow(img2[:,:,0]) + # plt.show() + else: + contours_only_text_parent_d_ordered = [] + contours_only_text_parent_d = [] + contours_only_text_parent = [] + + else: + contours_only_text_parent_d_ordered = [] + contours_only_text_parent_d = [] + contours_only_text_parent = [] + + else: + contours_only_text, hir_on_text = return_contours_of_image(text_only) + contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) + + if len(contours_only_text_parent) > 0: + areas_cnt_text = np.array([cv2.contourArea(contours_only_text_parent[j]) for j in range(len(contours_only_text_parent))]) + areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) + + contours_biggest = contours_only_text_parent[np.argmax(areas_cnt_text)] + contours_only_text_parent = [contours_only_text_parent[jz] for jz in range(len(contours_only_text_parent)) if areas_cnt_text[jz] > min_con_area] + areas_cnt_text_parent = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > min_con_area] + + index_con_parents = np.argsort(areas_cnt_text_parent) + contours_only_text_parent = list(np.array(contours_only_text_parent)[index_con_parents]) + areas_cnt_text_parent = list(np.array(areas_cnt_text_parent)[index_con_parents]) + + cx_bigest_big, cy_biggest_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest]) + cx_bigest, cy_biggest, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent) + self.logger.debug('areas_cnt_text_parent %s', areas_cnt_text_parent) + # self.logger.debug('areas_cnt_text_parent_d %s', areas_cnt_text_parent_d) + # self.logger.debug('len(contours_only_text_parent) %s', len(contours_only_text_parent_d)) + else: + pass + txt_con_org = get_textregion_contours_in_org_image(contours_only_text_parent, self.image, slope_first) + boxes_text, _ = get_text_region_boxes_by_given_contours(contours_only_text_parent) + boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) + + if not self.curved_line: + slopes, all_found_texline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, index_by_text_par_con = self.get_slopes_and_deskew_new(txt_con_org, contours_only_text_parent, textline_mask_tot_ea, image_page_rotated, boxes_text, slope_deskew) + slopes_marginals, all_found_texline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, all_box_coord_marginals, _ = self.get_slopes_and_deskew_new(polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, image_page_rotated, boxes_marginals, slope_deskew) + else: + + scale_param = 1 + all_found_texline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_curved(txt_con_org, contours_only_text_parent, cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1), image_page_rotated, boxes_text, text_only, num_col_classifier, scale_param, slope_deskew) + all_found_texline_polygons = small_textlines_to_parent_adherence2(all_found_texline_polygons, textline_mask_tot_ea, num_col_classifier) + all_found_texline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_curved(polygons_of_marginals, polygons_of_marginals, cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1), image_page_rotated, boxes_marginals, text_only, num_col_classifier, scale_param, slope_deskew) + all_found_texline_polygons_marginals = small_textlines_to_parent_adherence2(all_found_texline_polygons_marginals, textline_mask_tot_ea, num_col_classifier) + K.clear_session() + if self.full_layout: + if np.abs(slope_deskew) >= SLOPE_THRESHOLD: + contours_only_text_parent_d_ordered = list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) + text_regions_p, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, slopes, slopes_h, contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered = check_any_text_region_in_model_one_is_main_or_header(text_regions_p, regions_fully, contours_only_text_parent, all_box_coord, all_found_texline_polygons, slopes, contours_only_text_parent_d_ordered) + else: + contours_only_text_parent_d_ordered = None + text_regions_p, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, slopes, slopes_h, contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered = check_any_text_region_in_model_one_is_main_or_header(text_regions_p, regions_fully, contours_only_text_parent, all_box_coord, all_found_texline_polygons, slopes, contours_only_text_parent_d_ordered) + + if self.plotter: + self.plotter.save_plot_of_layout(text_regions_p, image_page) + self.plotter.save_plot_of_layout_all(text_regions_p, image_page) + + K.clear_session() + + polygons_of_tabels = [] + pixel_img = 4 + polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img) + all_found_texline_polygons = adhere_drop_capital_region_into_corresponding_textline(text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, all_found_texline_polygons, all_found_texline_polygons_h, kernel=KERNEL, curved_line=self.curved_line) + + # print(len(contours_only_text_parent_h),len(contours_only_text_parent_h_d_ordered),'contours_only_text_parent_h') + pixel_lines = 6 + + if not self.headers_off: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines, contours_only_text_parent_h) + else: + _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines, contours_only_text_parent_h_d_ordered) + elif self.headers_off: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document(np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) + else: + _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document(np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), num_col_classifier, pixel_lines) + + # print(peaks_neg_fin,peaks_neg_fin_d,'num_col2') + # print(splitter_y_new,splitter_y_new_d,'num_col_classifier') + # print(matrix_of_lines_ch.shape,matrix_of_lines_ch_d.shape,'matrix_of_lines_ch') + + if num_col_classifier >= 3: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + regions_without_separators = regions_without_separators.astype(np.uint8) + regions_without_separators = cv2.erode(regions_without_separators[:, :], KERNEL, iterations=6) + random_pixels_for_image = np.random.randn(regions_without_separators.shape[0], regions_without_separators.shape[1]) + random_pixels_for_image[random_pixels_for_image < -0.5] = 0 + random_pixels_for_image[random_pixels_for_image != 0] = 1 + regions_without_separators[(random_pixels_for_image[:, :] == 1) & (text_regions_p[:, :] == 5)] = 1 + else: + regions_without_separators_d = regions_without_separators_d.astype(np.uint8) + regions_without_separators_d = cv2.erode(regions_without_separators_d[:, :], KERNEL, iterations=6) + random_pixels_for_image = np.random.randn(regions_without_separators_d.shape[0], regions_without_separators_d.shape[1]) + random_pixels_for_image[random_pixels_for_image < -0.5] = 0 + random_pixels_for_image[random_pixels_for_image != 0] = 1 + regions_without_separators_d[(random_pixels_for_image[:, :] == 1) & (text_regions_p_1_n[:, :] == 5)] = 1 + + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + boxes = return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts) + else: + boxes_d = return_boxes_of_images_by_order_of_reading_new(splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, num_col_classifier, erosion_hurts) + + if self.plotter: + self.plotter.write_images_into_directory(polygons_of_images, image_page) + + if self.full_layout: + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) + else: + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) + + pcgts = self.writer.build_pagexml_full_layout(contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_found_texline_polygons_h, all_box_coord, all_box_coord_h, polygons_of_images, polygons_of_tabels, polygons_of_drop_capitals, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_xml) + self.logger.info("Job done in %ss", str(time.time() - t0)) + return pcgts + else: + contours_only_text_parent_h = None + if np.abs(slope_deskew) < SLOPE_THRESHOLD: + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) + else: + contours_only_text_parent_d_ordered = list(np.array(contours_only_text_parent_d_ordered)[index_by_text_par_con]) + order_text_new, id_of_texts_tot = self.do_order_of_regions(contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) + pcgts = self.writer.build_pagexml_no_full_layout(txt_con_org, page_coord, order_text_new, id_of_texts_tot, all_found_texline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_xml) + self.logger.info("Job done in %ss", str(time.time() - t0)) + return pcgts diff --git a/qurator/eynollah/ocrd-tool.json b/qurator/eynollah/ocrd-tool.json new file mode 100644 index 0000000..a51e77a --- /dev/null +++ b/qurator/eynollah/ocrd-tool.json @@ -0,0 +1,49 @@ +{ + "version": "0.0.8", + "git_url": "https://github.com/qurator-spk/eynollah", + "tools": { + "ocrd-eynollah-segment": { + "executable": "ocrd-eynollah-segment", + "categories": ["Layout analysis"], + "description": "Segment page into regions and lines and do reading order detection with eynollah", + "input_file_grp": ["OCR-D-IMG", "OCR-D-SEG-PAGE", "OCR-D-GT-SEG-PAGE"], + "output_file_grp": ["OCR-D-SEG-LINE"], + "steps": ["layout/segmentation/region", "layout/segmentation/line"], + "parameters": { + "models": { + "type": "string", + "format": "file", + "cacheable": true, + "description": "Path to directory containing models to be used (See https://qurator-data.de/eynollah)", + "required": true + }, + "dpi": { + "type": "number", + "format": "float", + "description": "pixel density in dots per inch (overrides any meta-data in the images); ignored if <= 0 (with fall-back 230)", + "default": 0 + }, + "full_layout": { + "type": "boolean", + "default": true, + "description": "Try to detect all element subtypes, including drop-caps and headings" + }, + "curved_line": { + "type": "boolean", + "default": false, + "description": "try to return contour of textlines instead of just rectangle bounding box. Needs more processing time" + }, + "allow_scaling": { + "type": "boolean", + "default": false, + "description": "check the resolution against the number of detected columns and if needed, scale the image up or down during layout detection (heuristic to improve quality and performance)" + }, + "headers_off": { + "type": "boolean", + "default": false, + "description": "ignore the special role of headings during reading order detection" + } + } + } + } +} diff --git a/src/eynollah/ocrd_cli.py b/qurator/eynollah/ocrd_cli.py similarity index 100% rename from src/eynollah/ocrd_cli.py rename to qurator/eynollah/ocrd_cli.py diff --git a/src/eynollah/plot.py b/qurator/eynollah/plot.py similarity index 92% rename from src/eynollah/plot.py rename to qurator/eynollah/plot.py index 412ae5a..18a7c14 100644 --- a/src/eynollah/plot.py +++ b/qurator/eynollah/plot.py @@ -1,8 +1,5 @@ -try: - import matplotlib.pyplot as plt - import matplotlib.patches as mpatches -except ImportError: - plt = mpatches = None +import matplotlib.pyplot as plt +import matplotlib.patches as mpatches import numpy as np import os.path import cv2 @@ -20,9 +17,7 @@ class EynollahPlotter(): def __init__( self, *, - dir_out, dir_of_all, - dir_save_page, dir_of_deskewed, dir_of_layout, dir_of_cropped_images, @@ -31,9 +26,7 @@ class EynollahPlotter(): scale_x=1, scale_y=1, ): - self.dir_out = dir_out self.dir_of_all = dir_of_all - self.dir_save_page = dir_save_page self.dir_of_layout = dir_of_layout self.dir_of_cropped_images = dir_of_cropped_images self.dir_of_deskewed = dir_of_deskewed @@ -79,8 +72,8 @@ class EynollahPlotter(): if self.dir_of_layout is not None: values = np.unique(text_regions_p[:, :]) # pixels=['Background' , 'Main text' , 'Heading' , 'Marginalia' ,'Drop capitals' , 'Images' , 'Seperators' , 'Tables', 'Graphics'] - pixels = ["Background", "Main text", "Header", "Marginalia", "Drop capital", "Image", "Separator", "Tables"] - values_indexes = [0, 1, 2, 8, 4, 5, 6, 10] + pixels = ["Background", "Main text", "Header", "Marginalia", "Drop capital", "Image", "Separator"] + values_indexes = [0, 1, 2, 8, 4, 5, 6] plt.figure(figsize=(40, 40)) plt.rcParams["font.size"] = "40" im = plt.imshow(text_regions_p[:, :]) @@ -93,8 +86,8 @@ class EynollahPlotter(): if self.dir_of_all is not None: values = np.unique(text_regions_p[:, :]) # pixels=['Background' , 'Main text' , 'Heading' , 'Marginalia' ,'Drop capitals' , 'Images' , 'Seperators' , 'Tables', 'Graphics'] - pixels = ["Background", "Main text", "Header", "Marginalia", "Drop capital", "Image", "Separator", "Tables"] - values_indexes = [0, 1, 2, 8, 4, 5, 6, 10] + pixels = ["Background", "Main text", "Header", "Marginalia", "Drop capital", "Image", "Separator"] + values_indexes = [0, 1, 2, 8, 4, 5, 6] plt.figure(figsize=(80, 40)) plt.rcParams["font.size"] = "40" plt.subplot(1, 2, 1) @@ -132,11 +125,7 @@ class EynollahPlotter(): def save_page_image(self, image_page): if self.dir_of_all is not None: cv2.imwrite(os.path.join(self.dir_of_all, self.image_filename_stem + "_page.png"), image_page) - if self.dir_save_page is not None: - cv2.imwrite(os.path.join(self.dir_save_page, self.image_filename_stem + "_page.png"), image_page) - def save_enhanced_image(self, img_res): - cv2.imwrite(os.path.join(self.dir_out, self.image_filename_stem + "_enhanced.png"), img_res) - + def save_plot_of_textline_density(self, img_patch_org): if self.dir_of_all is not None: plt.figure(figsize=(80,40)) diff --git a/qurator/eynollah/processor.py b/qurator/eynollah/processor.py new file mode 100644 index 0000000..41b12ae --- /dev/null +++ b/qurator/eynollah/processor.py @@ -0,0 +1,67 @@ +from json import loads +from pkg_resources import resource_string +from tempfile import NamedTemporaryFile +from pathlib import Path +from os.path import join + +from PIL import Image + +from ocrd import Processor +from ocrd_modelfactory import page_from_file, exif_from_filename +from ocrd_models import OcrdFile, OcrdExif +from ocrd_models.ocrd_page import to_xml +from ocrd_utils import ( + getLogger, + MIMETYPE_PAGE, + assert_file_grp_cardinality, + make_file_id +) + +from .eynollah import Eynollah +from .utils.pil_cv2 import pil2cv + +OCRD_TOOL = loads(resource_string(__name__, 'ocrd-tool.json').decode('utf8')) + +class EynollahProcessor(Processor): + + def __init__(self, *args, **kwargs): + kwargs['ocrd_tool'] = OCRD_TOOL['tools']['ocrd-eynollah-segment'] + kwargs['version'] = OCRD_TOOL['version'] + super().__init__(*args, **kwargs) + + def process(self): + LOG = getLogger('eynollah') + assert_file_grp_cardinality(self.input_file_grp, 1) + assert_file_grp_cardinality(self.output_file_grp, 1) + for n, input_file in enumerate(self.input_files): + page_id = input_file.pageId or input_file.ID + LOG.info("INPUT FILE %s (%d/%d) ", page_id, n + 1, len(self.input_files)) + pcgts = page_from_file(self.workspace.download_file(input_file)) + LOG.debug('width %s height %s', pcgts.get_Page().imageWidth, pcgts.get_Page().imageHeight) + self.add_metadata(pcgts) + page = pcgts.get_Page() + # XXX loses DPI information + # page_image, _, _ = self.workspace.image_from_page(page, page_id, feature_filter='binarized') + image_filename = self.workspace.download_file(next(self.workspace.mets.find_files(url=page.imageFilename))).local_filename + eynollah_kwargs = { + 'dir_models': self.resolve_resource(self.parameter['models']), + 'allow_enhancement': False, + 'curved_line': self.parameter['curved_line'], + 'full_layout': self.parameter['full_layout'], + 'allow_scaling': self.parameter['allow_scaling'], + 'headers_off': self.parameter['headers_off'], + 'override_dpi': self.parameter['dpi'], + 'logger': LOG, + 'pcgts': pcgts, + 'image_filename': image_filename + } + Eynollah(**eynollah_kwargs).run() + file_id = make_file_id(input_file, self.output_file_grp) + pcgts.set_pcGtsId(file_id) + self.workspace.add_file( + ID=file_id, + file_grp=self.output_file_grp, + pageId=page_id, + mimetype=MIMETYPE_PAGE, + local_filename=join(self.output_file_grp, file_id) + '.xml', + content=to_xml(pcgts)) diff --git a/src/eynollah/utils/__init__.py b/qurator/eynollah/utils/__init__.py similarity index 59% rename from src/eynollah/utils/__init__.py rename to qurator/eynollah/utils/__init__.py index c5962f8..fb6b476 100644 --- a/src/eynollah/utils/__init__.py +++ b/qurator/eynollah/utils/__init__.py @@ -1,13 +1,10 @@ -import time import math -try: - import matplotlib.pyplot as plt -except ImportError: - plt = None +import matplotlib.pyplot as plt import numpy as np from shapely import geometry import cv2 +import imutils from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d @@ -17,9 +14,9 @@ from .contour import (contours_in_same_horizon, return_contours_of_image, return_parent_contours) -def return_x_start_end_mothers_childs_and_type_of_reading_order( - x_min_hor_some, x_max_hor_some, cy_hor_some, peak_points, cy_hor_diff): - +def return_x_start_end_mothers_childs_and_type_of_reading_order(x_min_hor_some,x_max_hor_some,cy_hor_some,peak_points,cy_hor_diff): + + x_start=[] x_end=[] kind=[]#if covers 2 and more than 2 columns set it to 1 otherwise 0 @@ -27,43 +24,51 @@ def return_x_start_end_mothers_childs_and_type_of_reading_order( y_sep=[] y_diff=[] new_main_sep_y=[] - + indexer=0 for i in range(len(x_min_hor_some)): starting=x_min_hor_some[i]-peak_points starting=starting[starting>=0] min_start=np.argmin(starting) + + ending=peak_points-x_max_hor_some[i] len_ending_neg=len(ending[ending<=0]) - + ending=ending[ending>0] max_end=np.argmin(ending)+len_ending_neg - + + if (max_end-min_start)>=2: if (max_end-min_start)==(len(peak_points)-1): new_main_sep_y.append(indexer) - + #print((max_end-min_start),len(peak_points),'(max_end-min_start)') y_sep.append(cy_hor_some[i]) y_diff.append(cy_hor_diff[i]) x_end.append(max_end) - + x_start.append( min_start) - + len_sep.append(max_end-min_start) if max_end==min_start+1: kind.append(0) else: kind.append(1) - + indexer+=1 - - x_start_returned = np.array(x_start, dtype=int) - x_end_returned = np.array(x_end, dtype=int) - y_sep_returned = np.array(y_sep, dtype=int) - y_diff_returned = np.array(y_diff, dtype=int) - - all_args_uniq = contours_in_same_horizon(y_sep_returned) + + + x_start_returned=np.copy(x_start) + x_end_returned=np.copy(x_end) + y_sep_returned=np.copy(y_sep) + y_diff_returned=np.copy(y_diff) + + + + + all_args_uniq=contours_in_same_horizon(y_sep_returned) + args_to_be_unified=[] y_unified=[] y_diff_unified=[] @@ -79,10 +84,7 @@ def return_x_start_end_mothers_childs_and_type_of_reading_order( y_sep_same_hor=np.array(y_sep_returned)[all_args_uniq[dd]] y_diff_same_hor=np.array(y_diff_returned)[all_args_uniq[dd]] #print('burda2') - if (x_s_same_hor[0]==x_e_same_hor[1]-1 or - x_s_same_hor[1]==x_e_same_hor[0]-1 and - x_s_same_hor[0]!=x_s_same_hor[1] and - x_e_same_hor[0]!=x_e_same_hor[1]): + if x_s_same_hor[0]==(x_e_same_hor[1]-1) or x_s_same_hor[1]==(x_e_same_hor[0]-1) and x_s_same_hor[0]!=x_s_same_hor[1] and x_e_same_hor[0]!=x_e_same_hor[1]: #print('burda3') for arg_in in all_args_uniq[dd]: #print(arg_in,'arg_in') @@ -91,85 +93,91 @@ def return_x_start_end_mothers_childs_and_type_of_reading_order( y_diff_selected=np.max(y_diff_same_hor) x_s_selected=np.min(x_s_same_hor) x_e_selected=np.max(x_e_same_hor) - + x_s_unified.append(x_s_selected) x_e_unified.append(x_e_selected) y_unified.append(y_selected) y_diff_unified.append(y_diff_selected) + + + #print(x_s_same_hor,'x_s_same_hor') #print(x_e_same_hor[:]-1,'x_e_same_hor') #print('#############################') + #print(x_s_unified,'y_selected') #print(x_e_unified,'x_s_selected') #print(y_unified,'x_e_same_hor') - + args_lines_not_unified=list( set(range(len(y_sep_returned)))-set(args_to_be_unified) ) + #print(args_lines_not_unified,'args_lines_not_unified') - + x_start_returned_not_unified=list( np.array(x_start_returned)[args_lines_not_unified] ) x_end_returned_not_unified=list( np.array(x_end_returned)[args_lines_not_unified] ) y_sep_returned_not_unified=list (np.array(y_sep_returned)[args_lines_not_unified] ) y_diff_returned_not_unified=list (np.array(y_diff_returned)[args_lines_not_unified] ) - + for dv in range(len(y_unified)): y_sep_returned_not_unified.append(y_unified[dv]) y_diff_returned_not_unified.append(y_diff_unified[dv]) x_start_returned_not_unified.append(x_s_unified[dv]) x_end_returned_not_unified.append(x_e_unified[dv]) - + #print(y_sep_returned,'y_sep_returned') #print(x_start_returned,'x_start_returned') #print(x_end_returned,'x_end_returned') - - x_start_returned = np.array(x_start_returned_not_unified, dtype=int) - x_end_returned = np.array(x_end_returned_not_unified, dtype=int) - y_sep_returned = np.array(y_sep_returned_not_unified, dtype=int) - y_diff_returned = np.array(y_diff_returned_not_unified, dtype=int) - + + x_start_returned=np.copy(x_start_returned_not_unified) + x_end_returned=np.copy(x_end_returned_not_unified) + y_sep_returned=np.copy(y_sep_returned_not_unified) + y_diff_returned=np.copy(y_diff_returned_not_unified) + + #print(y_sep_returned,'y_sep_returned2') #print(x_start_returned,'x_start_returned2') #print(x_end_returned,'x_end_returned2') #print(new_main_sep_y,'new_main_sep_y') - + #print(x_start,'x_start') #print(x_end,'x_end') if len(new_main_sep_y)>0: - + min_ys=np.min(y_sep) max_ys=np.max(y_sep) - + y_mains=[] y_mains.append(min_ys) y_mains_sep_ohne_grenzen=[] - + for ii in range(len(new_main_sep_y)): y_mains.append(y_sep[new_main_sep_y[ii]]) y_mains_sep_ohne_grenzen.append(y_sep[new_main_sep_y[ii]]) - + y_mains.append(max_ys) - + y_mains_sorted=np.sort(y_mains) diff=np.diff(y_mains_sorted) argm=np.argmax(diff) - + y_min_new=y_mains_sorted[argm] y_max_new=y_mains_sorted[argm+1] - + #print(y_min_new,'y_min_new') #print(y_max_new,'y_max_new') + + #print(y_sep[new_main_sep_y[0]],y_sep,'yseps') x_start=np.array(x_start) x_end=np.array(x_end) kind=np.array(kind) y_sep=np.array(y_sep) - if (y_min_new in y_mains_sep_ohne_grenzen and - y_max_new in y_mains_sep_ohne_grenzen): + if (y_min_new in y_mains_sep_ohne_grenzen) and (y_max_new in y_mains_sep_ohne_grenzen): x_start=x_start[(y_sep>y_min_new) & (y_sepy_min_new) & (y_sepy_min_new) & (y_sepy_min_new) & (y_sepy_min_new) & (y_sep<=y_max_new)] #print('burda1') @@ -177,8 +185,7 @@ def return_x_start_end_mothers_childs_and_type_of_reading_order( #print('burda2') kind=kind[(y_sep>y_min_new) & (y_sep<=y_max_new)] y_sep=y_sep[(y_sep>y_min_new) & (y_sep<=y_max_new)] - elif (y_min_new not in y_mains_sep_ohne_grenzen and - y_max_new in y_mains_sep_ohne_grenzen): + elif (y_min_new not in y_mains_sep_ohne_grenzen) and (y_max_new in y_mains_sep_ohne_grenzen): x_start=x_start[(y_sep>=y_min_new) & (y_sep=y_min_new) & (y_sep=y_min_new) & (y_sep1: #print(np.array(remained_sep_indexes),'np.array(remained_sep_indexes)') #print(np.array(mother),'mother') - remained_sep_indexes_without_mother = remained_sep_indexes[mother==0] - remained_sep_indexes_with_child_without_mother = remained_sep_indexes[(mother==0) & (child==1)] + remained_sep_indexes_without_mother=np.array(list(remained_sep_indexes))[np.array(mother)==0] + remained_sep_indexes_with_child_without_mother=np.array(list(remained_sep_indexes))[(np.array(mother)==0) & (np.array(child)==1)] #print(remained_sep_indexes_without_mother,'remained_sep_indexes_without_mother') - #print(remained_sep_indexes_without_mother,'remained_sep_indexes_without_mother') - - x_end_with_child_without_mother = x_end[remained_sep_indexes_with_child_without_mother] - x_start_with_child_without_mother = x_start[remained_sep_indexes_with_child_without_mother] - y_lines_with_child_without_mother = y_sep[remained_sep_indexes_with_child_without_mother] - + + + + x_end_with_child_without_mother=np.array(x_end)[np.array(remained_sep_indexes_with_child_without_mother)] + + x_start_with_child_without_mother=np.array(x_start)[np.array(remained_sep_indexes_with_child_without_mother)] + + y_lines_with_child_without_mother=np.array(y_sep)[np.array(remained_sep_indexes_with_child_without_mother)] + + reading_orther_type=0 - x_end_without_mother = x_end[remained_sep_indexes_without_mother] - x_start_without_mother = x_start[remained_sep_indexes_without_mother] - y_lines_without_mother = y_sep[remained_sep_indexes_without_mother] - + + + x_end_without_mother=np.array(x_end)[np.array(remained_sep_indexes_without_mother)] + x_start_without_mother=np.array(x_start)[np.array(remained_sep_indexes_without_mother)] + y_lines_without_mother=np.array(y_sep)[np.array(remained_sep_indexes_without_mother)] + if len(remained_sep_indexes_without_mother)>=2: for i in range(len(remained_sep_indexes_without_mother)-1): - nodes_i=set(range(x_start[remained_sep_indexes_without_mother[i]], - x_end[remained_sep_indexes_without_mother[i]] - # + 1 - )) + ##nodes_i=set(range(x_start[remained_sep_indexes_without_mother[i]],x_end[remained_sep_indexes_without_mother[i]]+1)) + nodes_i=set(range(x_start[remained_sep_indexes_without_mother[i]],x_end[remained_sep_indexes_without_mother[i]])) for j in range(i+1,len(remained_sep_indexes_without_mother)): - nodes_j=set(range(x_start[remained_sep_indexes_without_mother[j]], - x_end[remained_sep_indexes_without_mother[j]] - # + 1 - )) - set_diff = nodes_i - nodes_j - if set_diff != nodes_i: - reading_orther_type = 1 + #nodes_j=set(range(x_start[remained_sep_indexes_without_mother[j]],x_end[remained_sep_indexes_without_mother[j]]+1)) + nodes_j=set(range(x_start[remained_sep_indexes_without_mother[j]],x_end[remained_sep_indexes_without_mother[j]])) + + set_diff=nodes_i-nodes_j + + if set_diff!=nodes_i: + reading_orther_type=1 else: - reading_orther_type = 0 + reading_orther_type=0 #print(reading_orther_type,'javab') + #print(y_lines_with_child_without_mother,'y_lines_with_child_without_mother') #print(x_start_with_child_without_mother,'x_start_with_child_without_mother') #print(x_end_with_child_without_mother,'x_end_with_hild_without_mother') - - len_sep_with_child = len(child[child==1]) - + + len_sep_with_child=len(np.array(child)[np.array(child)==1]) + #print(len_sep_with_child,'len_sep_with_child') - there_is_sep_with_child = 0 - if len_sep_with_child >= 1: - there_is_sep_with_child = 1 + there_is_sep_with_child=0 + + if len_sep_with_child>=1: + there_is_sep_with_child=1 + #print(all_args_uniq,'all_args_uniq') #print(args_to_be_unified,'args_to_be_unified') - - return (reading_orther_type, - x_start_returned, - x_end_returned, - y_sep_returned, - y_diff_returned, - y_lines_without_mother, - x_start_without_mother, - x_end_without_mother, - there_is_sep_with_child, - y_lines_with_child_without_mother, - x_start_with_child_without_mother, - x_end_with_child_without_mother, - new_main_sep_y) - + + + return reading_orther_type,x_start_returned, x_end_returned ,y_sep_returned,y_diff_returned,y_lines_without_mother,x_start_without_mother,x_end_without_mother,there_is_sep_with_child,y_lines_with_child_without_mother,x_start_with_child_without_mother,x_end_with_child_without_mother def crop_image_inside_box(box, img_org_copy): image_box = img_org_copy[box[1] : box[1] + box[3], box[0] : box[0] + box[2]] return image_box, [box[1], box[1] + box[3], box[0], box[0] + box[2]] @@ -307,6 +304,7 @@ def otsu_copy_binary(img): img1 = img[:, :, 0] retval1, threshold1 = cv2.threshold(img1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + img_r[:, :, 0] = threshold1 img_r[:, :, 1] = threshold1 img_r[:, :, 2] = threshold1 @@ -314,7 +312,9 @@ def otsu_copy_binary(img): img_r = img_r / float(np.max(img_r)) * 255 return img_r + def find_features_of_lines(contours_main): + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] @@ -326,6 +326,7 @@ def find_features_of_lines(contours_main): y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) slope_lines = [] + for kk in range(len(contours_main)): [vx, vy, x, y] = cv2.fitLine(contours_main[kk], cv2.DIST_L2, 0, 0.01, 0.01) slope_lines.append(((vy / vx) / np.pi * 180)[0]) @@ -338,42 +339,29 @@ def find_features_of_lines(contours_main): slope_lines[(slope_lines != 0) & (slope_lines != 1)] = 2 dis_x = np.abs(x_max_main - x_min_main) - return (slope_lines, - dis_x, - x_min_main, - x_max_main, - np.array(cy_main), - np.array(slope_lines_org), - y_min_main, - y_max_main, - np.array(cx_main)) + return slope_lines, dis_x, x_min_main, x_max_main, np.array(cy_main), np.array(slope_lines_org), y_min_main, y_max_main, np.array(cx_main) def boosting_headers_by_longshot_region_segmentation(textregion_pre_p, textregion_pre_np, img_only_text): textregion_pre_p_org = np.copy(textregion_pre_p) # 4 is drop capitals - headers_in_longshot = textregion_pre_np[:, :, 0] == 2 - #headers_in_longshot = ((textregion_pre_np[:,:,0]==2) | - # (textregion_pre_np[:,:,0]==1)) - textregion_pre_p[:, :, 0][headers_in_longshot & - (textregion_pre_p[:, :, 0] != 4)] = 2 + headers_in_longshot = (textregion_pre_np[:, :, 0] == 2) * 1 + # headers_in_longshot= ( (textregion_pre_np[:,:,0]==2) | (textregion_pre_np[:,:,0]==1) )*1 + textregion_pre_p[:, :, 0][(headers_in_longshot[:, :] == 1) & (textregion_pre_p[:, :, 0] != 4)] = 2 textregion_pre_p[:, :, 0][textregion_pre_p[:, :, 0] == 1] = 0 # earlier it was so, but by this manner the drop capitals are also deleted - # textregion_pre_p[:,:,0][(img_only_text[:,:]==1) & - # (textregion_pre_p[:,:,0]!=7) & - # (textregion_pre_p[:,:,0]!=2)] = 1 - textregion_pre_p[:, :, 0][(img_only_text[:, :] == 1) & - (textregion_pre_p[:, :, 0] != 7) & - (textregion_pre_p[:, :, 0] != 4) & - (textregion_pre_p[:, :, 0] != 2)] = 1 + # textregion_pre_p[:,:,0][( img_only_text[:,:]==1) & (textregion_pre_p[:,:,0]!=7) & (textregion_pre_p[:,:,0]!=2)]=1 + textregion_pre_p[:, :, 0][(img_only_text[:, :] == 1) & (textregion_pre_p[:, :, 0] != 7) & (textregion_pre_p[:, :, 0] != 4) & (textregion_pre_p[:, :, 0] != 2)] = 1 return textregion_pre_p + def find_num_col_deskew(regions_without_separators, sigma_, multiplier=3.8): - regions_without_separators_0 = regions_without_separators.sum(axis=1) + regions_without_separators_0 = regions_without_separators[:,:].sum(axis=1) z = gaussian_filter1d(regions_without_separators_0, sigma_) return np.std(z) -def find_num_col(regions_without_separators, num_col_classifier, tables, multiplier=3.8): - regions_without_separators_0 = regions_without_separators.sum(axis=0) + +def find_num_col(regions_without_separators, multiplier=3.8): + regions_without_separators_0 = regions_without_separators[:, :].sum(axis=0) ##plt.plot(regions_without_separators_0) ##plt.show() sigma_ = 35 # 70#35 @@ -384,7 +372,7 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl y = regions_without_separators_0 # [first_nonzero:last_nonzero] y_help = np.zeros(len(y) + 20) y_help[10 : len(y) + 10] = y - x = np.arange(len(y)) + x = np.array(range(len(y))) zneg_rev = -y_help + np.max(y_help) zneg = np.zeros(len(zneg_rev) + 20) zneg[10 : len(zneg_rev) + 10] = zneg_rev @@ -398,12 +386,9 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl last_nonzero = last_nonzero - 100 first_nonzero = first_nonzero + 200 - peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & - (peaks_neg < last_nonzero)] - peaks = peaks[(peaks > 0.06 * regions_without_separators.shape[1]) & - (peaks < 0.94 * regions_without_separators.shape[1])] - peaks_neg = peaks_neg[(peaks_neg > 370) & - (peaks_neg < (regions_without_separators.shape[1] - 370))] + peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & (peaks_neg < last_nonzero)] + peaks = peaks[(peaks > 0.06 * regions_without_separators.shape[1]) & (peaks < 0.94 * regions_without_separators.shape[1])] + peaks_neg = peaks_neg[(peaks_neg > 370) & (peaks_neg < (regions_without_separators.shape[1] - 370))] interest_pos = z[peaks] interest_pos = interest_pos[interest_pos > 10] # plt.plot(z) @@ -420,8 +405,7 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl # print(np.min(interest_pos),np.max(interest_pos),np.max(interest_pos)/np.min(interest_pos),'minmax') dis_talaei = (min_peaks_pos - min_peaks_neg) / multiplier - grenze = min_peaks_pos - dis_talaei - # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 + grenze = min_peaks_pos - dis_talaei # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 # print(interest_neg,'interest_neg') # print(grenze,'grenze') @@ -433,19 +417,6 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl peaks_neg_fin = peaks_neg[(interest_neg < grenze)] # interest_neg_fin=interest_neg[(interest_neg= 3: - index_sort_interest_neg_fin= np.argsort(interest_neg_fin) - peaks_neg_sorted = np.array(peaks_neg)[index_sort_interest_neg_fin] - interest_neg_fin_sorted = np.array(interest_neg_fin)[index_sort_interest_neg_fin] - - if len(index_sort_interest_neg_fin)>=num_col_classifier: - peaks_neg_fin = list( peaks_neg_sorted[:num_col_classifier] ) - interest_neg_fin = list( interest_neg_fin_sorted[:num_col_classifier] ) - else: - peaks_neg_fin = peaks_neg[:] - interest_neg_fin = interest_neg[:] - num_col = (len(interest_neg_fin)) + 1 # print(peaks_neg_fin,'peaks_neg_fin') @@ -457,26 +428,19 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl p_g_u = len(y) - int(len(y) / 4.0) if num_col == 3: - if ((peaks_neg_fin[0] > p_g_u and - peaks_neg_fin[1] > p_g_u) or - (peaks_neg_fin[0] < p_g_l and - peaks_neg_fin[1] < p_g_l) or - (peaks_neg_fin[0] + 200 < p_m and - peaks_neg_fin[1] < p_m) or - (peaks_neg_fin[0] - 200 > p_m and - peaks_neg_fin[1] > p_m)): + if (peaks_neg_fin[0] > p_g_u and peaks_neg_fin[1] > p_g_u) or (peaks_neg_fin[0] < p_g_l and peaks_neg_fin[1] < p_g_l) or ((peaks_neg_fin[0] + 200) < p_m and peaks_neg_fin[1] < p_m) or ((peaks_neg_fin[0] - 200) > p_m and peaks_neg_fin[1] > p_m): num_col = 1 peaks_neg_fin = [] if num_col == 2: - if (peaks_neg_fin[0] > p_g_u or - peaks_neg_fin[0] < p_g_l): + if (peaks_neg_fin[0] > p_g_u) or (peaks_neg_fin[0] < p_g_l): num_col = 1 peaks_neg_fin = [] ##print(len(peaks_neg_fin)) diff_peaks = np.abs(np.diff(peaks_neg_fin)) + cut_off = 400 peaks_neg_true = [] forest = [] @@ -512,35 +476,23 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl ##print(num_col,'early') if num_col == 3: - if ((peaks_neg_true[0] > p_g_u and - peaks_neg_true[1] > p_g_u) or - (peaks_neg_true[0] < p_g_l and - peaks_neg_true[1] < p_g_l) or - (peaks_neg_true[0] < p_m and - peaks_neg_true[1] + 200 < p_m) or - (peaks_neg_true[0] - 200 > p_m and - peaks_neg_true[1] > p_m)): + if (peaks_neg_true[0] > p_g_u and peaks_neg_true[1] > p_g_u) or (peaks_neg_true[0] < p_g_l and peaks_neg_true[1] < p_g_l) or (peaks_neg_true[0] < p_m and (peaks_neg_true[1] + 200) < p_m) or ((peaks_neg_true[0] - 200) > p_m and peaks_neg_true[1] > p_m): num_col = 1 peaks_neg_true = [] - elif (peaks_neg_true[0] < p_g_u and - peaks_neg_true[0] > p_g_l and - peaks_neg_true[1] > p_u_quarter): + elif (peaks_neg_true[0] < p_g_u and peaks_neg_true[0] > p_g_l) and (peaks_neg_true[1] > p_u_quarter): peaks_neg_true = [peaks_neg_true[0]] - elif (peaks_neg_true[1] < p_g_u and - peaks_neg_true[1] > p_g_l and - peaks_neg_true[0] < p_quarter): + elif (peaks_neg_true[1] < p_g_u and peaks_neg_true[1] > p_g_l) and (peaks_neg_true[0] < p_quarter): peaks_neg_true = [peaks_neg_true[1]] if num_col == 2: - if (peaks_neg_true[0] > p_g_u or - peaks_neg_true[0] < p_g_l): + if (peaks_neg_true[0] > p_g_u) or (peaks_neg_true[0] < p_g_l): num_col = 1 peaks_neg_true = [] - diff_peaks_abnormal = diff_peaks[diff_peaks < 360] + diff_peaks_annormal = diff_peaks[diff_peaks < 360] - if len(diff_peaks_abnormal) > 0: - arg_help = np.arange(len(diff_peaks)) + if len(diff_peaks_annormal) > 0: + arg_help = np.array(range(len(diff_peaks))) arg_help_ann = arg_help[diff_peaks < 360] peaks_neg_fin_new = [] @@ -562,6 +514,7 @@ def find_num_col(regions_without_separators, num_col_classifier, tables, multipl # plt.plot(peaks_neg_true,z[peaks_neg_true],'*') # plt.plot([0,len(y)], [grenze,grenze]) # plt.show() + ##print(len(peaks_neg_true)) return len(peaks_neg_true), peaks_neg_true @@ -570,6 +523,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): ##plt.plot(regions_without_separators_0) ##plt.show() + sigma_ = 15 meda_n_updown = regions_without_separators_0[len(regions_without_separators_0) :: -1] @@ -580,24 +534,32 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): last_nonzero = len(regions_without_separators_0) - last_nonzero y = regions_without_separators_0 # [first_nonzero:last_nonzero] + y_help = np.zeros(len(y) + 20) + y_help[10 : len(y) + 10] = y - x = np.arange(len(y)) + + x = np.array(range(len(y))) zneg_rev = -y_help + np.max(y_help) + zneg = np.zeros(len(zneg_rev) + 20) + zneg[10 : len(zneg_rev) + 10] = zneg_rev + z = gaussian_filter1d(y, sigma_) zneg = gaussian_filter1d(zneg, sigma_) peaks_neg, _ = find_peaks(zneg, height=0) peaks, _ = find_peaks(z, height=0) + peaks_neg = peaks_neg - 10 - 10 + peaks_neg_org = np.copy(peaks_neg) - peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & - (peaks_neg < last_nonzero)] - peaks = peaks[(peaks > 0.09 * regions_without_separators.shape[1]) & - (peaks < 0.91 * regions_without_separators.shape[1])] + + peaks_neg = peaks_neg[(peaks_neg > first_nonzero) & (peaks_neg < last_nonzero)] + + peaks = peaks[(peaks > 0.09 * regions_without_separators.shape[1]) & (peaks < 0.91 * regions_without_separators.shape[1])] peaks_neg = peaks_neg[(peaks_neg > 500) & (peaks_neg < (regions_without_separators.shape[1] - 500))] # print(peaks) @@ -612,8 +574,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): # $print(min_peaks_pos) dis_talaei = (min_peaks_pos - min_peaks_neg) / multiplier # print(interest_pos) - grenze = min_peaks_pos - dis_talaei - # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 + grenze = min_peaks_pos - dis_talaei # np.mean(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])-np.std(y[peaks_neg[0]:peaks_neg[len(peaks_neg)-1]])/2.0 interest_neg_fin = interest_neg[(interest_neg < grenze)] peaks_neg_fin = peaks_neg[(interest_neg < grenze)] @@ -627,21 +588,13 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): p_g_u = len(y) - int(len(y) / 3.0) if num_col == 3: - if ((peaks_neg_fin[0] > p_g_u and - peaks_neg_fin[1] > p_g_u) or - (peaks_neg_fin[0] < p_g_l and - peaks_neg_fin[1] < p_g_l) or - (peaks_neg_fin[0] < p_m and - peaks_neg_fin[1] < p_m) or - (peaks_neg_fin[0] > p_m and - peaks_neg_fin[1] > p_m)): + if (peaks_neg_fin[0] > p_g_u and peaks_neg_fin[1] > p_g_u) or (peaks_neg_fin[0] < p_g_l and peaks_neg_fin[1] < p_g_l) or (peaks_neg_fin[0] < p_m and peaks_neg_fin[1] < p_m) or (peaks_neg_fin[0] > p_m and peaks_neg_fin[1] > p_m): num_col = 1 else: pass if num_col == 2: - if (peaks_neg_fin[0] > p_g_u or - peaks_neg_fin[0] < p_g_l): + if (peaks_neg_fin[0] > p_g_u) or (peaks_neg_fin[0] < p_g_l): num_col = 1 else: pass @@ -680,36 +633,23 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): p_u_quarter = len(y) - p_quarter if num_col == 3: - if ((peaks_neg_true[0] > p_g_u and - peaks_neg_true[1] > p_g_u) or - (peaks_neg_true[0] < p_g_l and - peaks_neg_true[1] < p_g_l) or - (peaks_neg_true[0] < p_m and - peaks_neg_true[1] < p_m) or - (peaks_neg_true[0] > p_m and - peaks_neg_true[1] > p_m)): + if (peaks_neg_true[0] > p_g_u and peaks_neg_true[1] > p_g_u) or (peaks_neg_true[0] < p_g_l and peaks_neg_true[1] < p_g_l) or (peaks_neg_true[0] < p_m and peaks_neg_true[1] < p_m) or (peaks_neg_true[0] > p_m and peaks_neg_true[1] > p_m): num_col = 1 peaks_neg_true = [] - elif (peaks_neg_true[0] < p_g_u and - peaks_neg_true[0] > p_g_l and - peaks_neg_true[1] > p_u_quarter): + elif (peaks_neg_true[0] < p_g_u and peaks_neg_true[0] > p_g_l) and (peaks_neg_true[1] > p_u_quarter): peaks_neg_true = [peaks_neg_true[0]] - elif (peaks_neg_true[1] < p_g_u and - peaks_neg_true[1] > p_g_l and - peaks_neg_true[0] < p_quarter): + elif (peaks_neg_true[1] < p_g_u and peaks_neg_true[1] > p_g_l) and (peaks_neg_true[0] < p_quarter): peaks_neg_true = [peaks_neg_true[1]] else: pass if num_col == 2: - if (peaks_neg_true[0] > p_g_u or - peaks_neg_true[0] < p_g_l): + if (peaks_neg_true[0] > p_g_u) or (peaks_neg_true[0] < p_g_l): num_col = 1 peaks_neg_true = [] if num_col == 4: - if (len(np.array(peaks_neg_true)[np.array(peaks_neg_true) < p_g_l]) == 2 or - len(np.array(peaks_neg_true)[np.array(peaks_neg_true) > (len(y) - p_g_l)]) == 2): + if len(np.array(peaks_neg_true)[np.array(peaks_neg_true) < p_g_l]) == 2 or len(np.array(peaks_neg_true)[np.array(peaks_neg_true) > (len(y) - p_g_l)]) == 2: num_col = 1 peaks_neg_true = [] else: @@ -721,10 +661,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): for i in range(len(peaks_neg_true)): hill_main = peaks_neg_true[i] # deep_depth=z[peaks_neg] - hills_around = peaks_neg_org[((peaks_neg_org > hill_main) & - (peaks_neg_org <= hill_main + 400)) | - ((peaks_neg_org < hill_main) & - (peaks_neg_org >= hill_main - 400))] + hills_around = peaks_neg_org[((peaks_neg_org > hill_main) & (peaks_neg_org <= hill_main + 400)) | ((peaks_neg_org < hill_main) & (peaks_neg_org >= hill_main - 400))] deep_depth_around = z[hills_around] # print(hill_main,z[hill_main],hills_around,deep_depth_around,'manoooo') @@ -737,11 +674,13 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): pass diff_peaks_annormal = diff_peaks[diff_peaks < 360] + if len(diff_peaks_annormal) > 0: - arg_help = np.arange(len(diff_peaks)) + arg_help = np.array(range(len(diff_peaks))) arg_help_ann = arg_help[diff_peaks < 360] peaks_neg_fin_new = [] + for ii in range(len(peaks_neg_fin)): if ii in arg_help_ann: arg_min = np.argmin([interest_neg_fin[ii], interest_neg_fin[ii + 1]]) @@ -749,6 +688,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): peaks_neg_fin_new.append(peaks_neg_fin[ii]) else: peaks_neg_fin_new.append(peaks_neg_fin[ii + 1]) + elif (ii - 1) in arg_help_ann: pass else: @@ -758,6 +698,7 @@ def find_num_col_only_image(regions_without_separators, multiplier=3.8): # sometime pages with one columns gives also some negative peaks. delete those peaks param = z[peaks_neg_true] / float(min_peaks_pos) * 100 + if len(param[param <= 41]) == 0: peaks_neg_true = [] @@ -768,9 +709,11 @@ def find_num_col_by_vertical_lines(regions_without_separators, multiplier=3.8): ##plt.plot(regions_without_separators_0) ##plt.show() + sigma_ = 35 # 70#35 z = gaussian_filter1d(regions_without_separators_0, sigma_) + peaks, _ = find_peaks(z, height=0) # print(peaks,'peaksnew') @@ -778,43 +721,39 @@ def find_num_col_by_vertical_lines(regions_without_separators, multiplier=3.8): def return_regions_without_separators(regions_pre): kernel = np.ones((5, 5), np.uint8) - regions_without_separators = ((regions_pre[:, :] != 6) & - (regions_pre[:, :] != 0)) - # regions_without_separators=( (image_regions_eraly_p[:,:,:]!=6) & - # (image_regions_eraly_p[:,:,:]!=0) & - # (image_regions_eraly_p[:,:,:]!=5) & - # (image_regions_eraly_p[:,:,:]!=8) & - # (image_regions_eraly_p[:,:,:]!=7)) + regions_without_separators = ((regions_pre[:, :] != 6) & (regions_pre[:, :] != 0)) * 1 + # regions_without_separators=( (image_regions_eraly_p[:,:,:]!=6) & (image_regions_eraly_p[:,:,:]!=0) & (image_regions_eraly_p[:,:,:]!=5) & (image_regions_eraly_p[:,:,:]!=8) & (image_regions_eraly_p[:,:,:]!=7))*1 - regions_without_separators = cv2.erode(regions_without_separators.astype(np.uint8), kernel, iterations=6) + regions_without_separators = regions_without_separators.astype(np.uint8) + + regions_without_separators = cv2.erode(regions_without_separators, kernel, iterations=6) return regions_without_separators + def put_drop_out_from_only_drop_model(layout_no_patch, layout1): + drop_only = (layout_no_patch[:, :, 0] == 4) * 1 contours_drop, hir_on_drop = return_contours_of_image(drop_only) contours_drop_parent = return_parent_contours(contours_drop, hir_on_drop) - areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) - for j in range(len(contours_drop_parent))]) + areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) areas_cnt_text = areas_cnt_text / float(drop_only.shape[0] * drop_only.shape[1]) - contours_drop_parent = [contours_drop_parent[jz] - for jz in range(len(contours_drop_parent)) - if areas_cnt_text[jz] > 0.00001] - areas_cnt_text = [areas_cnt_text[jz] - for jz in range(len(areas_cnt_text)) - if areas_cnt_text[jz] > 0.00001] + + contours_drop_parent = [contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz] > 0.00001] + + areas_cnt_text = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > 0.00001] contours_drop_parent_final = [] + for jj in range(len(contours_drop_parent)): x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) # boxes.append([int(x), int(y), int(w), int(h)]) map_of_drop_contour_bb = np.zeros((layout1.shape[0], layout1.shape[1])) map_of_drop_contour_bb[y : y + h, x : x + w] = layout1[y : y + h, x : x + w] - if (100. * - (map_of_drop_contour_bb == 1).sum() / - (map_of_drop_contour_bb == 5).sum()) >= 15: + + if (((map_of_drop_contour_bb == 1) * 1).sum() / float(((map_of_drop_contour_bb == 5) * 1).sum()) * 100) >= 15: contours_drop_parent_final.append(contours_drop_parent[jj]) layout_no_patch[:, :, 0][layout_no_patch[:, :, 0] == 4] = 0 @@ -823,60 +762,37 @@ def put_drop_out_from_only_drop_model(layout_no_patch, layout1): return layout_no_patch -def putt_bb_of_drop_capitals_of_model_in_patches_in_layout(layout_in_patch, drop_capital_label, text_regions_p): - drop_only = (layout_in_patch[:, :, 0] == drop_capital_label) * 1 +def putt_bb_of_drop_capitals_of_model_in_patches_in_layout(layout_in_patch): + + drop_only = (layout_in_patch[:, :, 0] == 4) * 1 contours_drop, hir_on_drop = return_contours_of_image(drop_only) contours_drop_parent = return_parent_contours(contours_drop, hir_on_drop) - areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) - for j in range(len(contours_drop_parent))]) + areas_cnt_text = np.array([cv2.contourArea(contours_drop_parent[j]) for j in range(len(contours_drop_parent))]) areas_cnt_text = areas_cnt_text / float(drop_only.shape[0] * drop_only.shape[1]) - contours_drop_parent = [contours_drop_parent[jz] - for jz in range(len(contours_drop_parent)) - if areas_cnt_text[jz] > 0.00001] - areas_cnt_text = [areas_cnt_text[jz] - for jz in range(len(areas_cnt_text)) - if areas_cnt_text[jz] > 0.00001] + + contours_drop_parent = [contours_drop_parent[jz] for jz in range(len(contours_drop_parent)) if areas_cnt_text[jz] > 0.00001] + + areas_cnt_text = [areas_cnt_text[jz] for jz in range(len(areas_cnt_text)) if areas_cnt_text[jz] > 0.001] contours_drop_parent_final = [] + for jj in range(len(contours_drop_parent)): x, y, w, h = cv2.boundingRect(contours_drop_parent[jj]) - box = slice(y, y + h), slice(x, x + w) - box0 = box + (0,) - mask_of_drop_cpaital_in_early_layout = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1])) - mask_of_drop_cpaital_in_early_layout[box] = text_regions_p[box] - - all_drop_capital_pixels_which_is_text_in_early_lo = np.sum(mask_of_drop_cpaital_in_early_layout[box]==1) - mask_of_drop_cpaital_in_early_layout[box] = 1 - all_drop_capital_pixels = np.sum(mask_of_drop_cpaital_in_early_layout==1) - - percent_text_to_all_in_drop = all_drop_capital_pixels_which_is_text_in_early_lo / float(all_drop_capital_pixels) - if (areas_cnt_text[jj] * float(drop_only.shape[0] * drop_only.shape[1]) / float(w * h) > 0.6 and - percent_text_to_all_in_drop >= 0.3): - layout_in_patch[box0] = drop_capital_label - else: - layout_in_patch[box0][layout_in_patch[box0] == drop_capital_label] = drop_capital_label - layout_in_patch[box0][layout_in_patch[box0] == 0] = drop_capital_label - layout_in_patch[box0][layout_in_patch[box0] == 4] = drop_capital_label# images - #layout_in_patch[box0][layout_in_patch[box0] == drop_capital_label] = 1#drop_capital_label + layout_in_patch[y : y + h, x : x + w, 0] = 4 return layout_in_patch -def check_any_text_region_in_model_one_is_main_or_header( - regions_model_1, regions_model_full, - contours_only_text_parent, - all_box_coord, all_found_textline_polygons, - slopes, - contours_only_text_parent_d_ordered, conf_contours): - - cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin = \ - find_new_features_of_contours(contours_only_text_parent) +def check_any_text_region_in_model_one_is_main_or_header(regions_model_1,regions_model_full,contours_only_text_parent,all_box_coord,all_found_texline_polygons,slopes,contours_only_text_parent_d_ordered): + cx_main,cy_main ,x_min_main , x_max_main, y_min_main ,y_max_main,y_corr_x_min_from_argmin=find_new_features_of_contours(contours_only_text_parent) length_con=x_max_main-x_min_main height_con=y_max_main-y_min_main - all_found_textline_polygons_main=[] - all_found_textline_polygons_head=[] + + + all_found_texline_polygons_main=[] + all_found_texline_polygons_head=[] all_box_coord_main=[] all_box_coord_head=[] @@ -887,9 +803,6 @@ def check_any_text_region_in_model_one_is_main_or_header( contours_only_text_parent_main=[] contours_only_text_parent_head=[] - conf_contours_main=[] - conf_contours_head=[] - contours_only_text_parent_main_d=[] contours_only_text_parent_head_d=[] @@ -898,10 +811,14 @@ def check_any_text_region_in_model_one_is_main_or_header( img=np.zeros((regions_model_1.shape[0],regions_model_1.shape[1],3)) img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) + + all_pixels=((img[:,:,0]==255)*1).sum() + pixels_header=( ( (img[:,:,0]==255) & (regions_model_full[:,:,0]==2) )*1 ).sum() pixels_main=all_pixels-pixels_header + if (pixels_header>=pixels_main) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 contours_only_text_parent_head.append(con) @@ -909,136 +826,28 @@ def check_any_text_region_in_model_one_is_main_or_header( contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_head.append(all_box_coord[ii]) slopes_head.append(slopes[ii]) - all_found_textline_polygons_head.append(all_found_textline_polygons[ii]) - conf_contours_head.append(None) + all_found_texline_polygons_head.append(all_found_texline_polygons[ii]) else: regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 contours_only_text_parent_main.append(con) - conf_contours_main.append(conf_contours[ii]) if contours_only_text_parent_d_ordered is not None: contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) all_box_coord_main.append(all_box_coord[ii]) slopes_main.append(slopes[ii]) - all_found_textline_polygons_main.append(all_found_textline_polygons[ii]) + all_found_texline_polygons_main.append(all_found_texline_polygons[ii]) #print(all_pixels,pixels_main,pixels_header) - return (regions_model_1, - contours_only_text_parent_main, - contours_only_text_parent_head, - all_box_coord_main, - all_box_coord_head, - all_found_textline_polygons_main, - all_found_textline_polygons_head, - slopes_main, - slopes_head, - contours_only_text_parent_main_d, - contours_only_text_parent_head_d, - conf_contours_main, - conf_contours_head) -def check_any_text_region_in_model_one_is_main_or_header_light( - regions_model_1, regions_model_full, - contours_only_text_parent, - all_box_coord, all_found_textline_polygons, - slopes, - contours_only_text_parent_d_ordered, - conf_contours): - ### to make it faster - h_o = regions_model_1.shape[0] - w_o = regions_model_1.shape[1] - zoom = 3 - regions_model_1 = cv2.resize(regions_model_1, (regions_model_1.shape[1] // zoom, - regions_model_1.shape[0] // zoom), - interpolation=cv2.INTER_NEAREST) - regions_model_full = cv2.resize(regions_model_full, (regions_model_full.shape[1] // zoom, - regions_model_full.shape[0] // zoom), - interpolation=cv2.INTER_NEAREST) - contours_only_text_parent = [(i / zoom).astype(int) for i in contours_only_text_parent] - - ### - cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin = \ - find_new_features_of_contours(contours_only_text_parent) - - length_con=x_max_main-x_min_main - height_con=y_max_main-y_min_main - - all_found_textline_polygons_main=[] - all_found_textline_polygons_head=[] - - all_box_coord_main=[] - all_box_coord_head=[] - - slopes_main=[] - slopes_head=[] - - contours_only_text_parent_main=[] - contours_only_text_parent_head=[] - - conf_contours_main=[] - conf_contours_head=[] - - contours_only_text_parent_main_d=[] - contours_only_text_parent_head_d=[] - - for ii in range(len(contours_only_text_parent)): - con=contours_only_text_parent[ii] - img=np.zeros((regions_model_1.shape[0], regions_model_1.shape[1], 3)) - img = cv2.fillPoly(img, pts=[con], color=(255, 255, 255)) - - all_pixels = (img[:,:,0]==255).sum() - pixels_header=((img[:,:,0]==255) & - (regions_model_full[:,:,0]==2)).sum() - pixels_main = all_pixels - pixels_header - - if (pixels_header/float(pixels_main)>=0.3) and ( (length_con[ii]/float(height_con[ii]) )>=1.3 ): - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=2 - contours_only_text_parent_head.append(con) - if contours_only_text_parent_d_ordered is not None: - contours_only_text_parent_head_d.append(contours_only_text_parent_d_ordered[ii]) - all_box_coord_head.append(all_box_coord[ii]) - slopes_head.append(slopes[ii]) - all_found_textline_polygons_head.append(all_found_textline_polygons[ii]) - conf_contours_head.append(None) - else: - regions_model_1[:,:][(regions_model_1[:,:]==1) & (img[:,:,0]==255) ]=1 - contours_only_text_parent_main.append(con) - conf_contours_main.append(conf_contours[ii]) - if contours_only_text_parent_d_ordered is not None: - contours_only_text_parent_main_d.append(contours_only_text_parent_d_ordered[ii]) - all_box_coord_main.append(all_box_coord[ii]) - slopes_main.append(slopes[ii]) - all_found_textline_polygons_main.append(all_found_textline_polygons[ii]) - - #print(all_pixels,pixels_main,pixels_header) - - ### to make it faster - regions_model_1 = cv2.resize(regions_model_1, (w_o, h_o), interpolation=cv2.INTER_NEAREST) - # regions_model_full = cv2.resize(img, (regions_model_full.shape[1] // zoom, - # regions_model_full.shape[0] // zoom), - # interpolation=cv2.INTER_NEAREST) - contours_only_text_parent_head = [(i * zoom).astype(int) for i in contours_only_text_parent_head] - contours_only_text_parent_main = [(i * zoom).astype(int) for i in contours_only_text_parent_main] - ### - - return (regions_model_1, - contours_only_text_parent_main, - contours_only_text_parent_head, - all_box_coord_main, - all_box_coord_head, - all_found_textline_polygons_main, - all_found_textline_polygons_head, - slopes_main, - slopes_head, - contours_only_text_parent_main_d, - contours_only_text_parent_head_d, - conf_contours_main, - conf_contours_head) + #plt.imshow(img[:,:,0]) + #plt.show() + return regions_model_1,contours_only_text_parent_main,contours_only_text_parent_head,all_box_coord_main,all_box_coord_head,all_found_texline_polygons_main,all_found_texline_polygons_head,slopes_main,slopes_head,contours_only_text_parent_main_d,contours_only_text_parent_head_d def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col): # print(textlines_con) # textlines_con=textlines_con.astype(np.uint32) + textlines_con_changed = [] for m1 in range(len(textlines_con)): @@ -1057,10 +866,9 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) ##plt.imshow(img_text_all) ##plt.show() - areas_cnt_text = np.array([cv2.contourArea(textlines_tot[j]) - for j in range(len(textlines_tot))]) + areas_cnt_text = np.array([cv2.contourArea(textlines_tot[j]) for j in range(len(textlines_tot))]) areas_cnt_text = areas_cnt_text / float(textline_iamge.shape[0] * textline_iamge.shape[1]) - indexes_textlines = np.arange(len(textlines_tot)) + indexes_textlines = np.array(range(len(textlines_tot))) # print(areas_cnt_text,np.min(areas_cnt_text),np.max(areas_cnt_text)) if num_col == 0: @@ -1095,7 +903,9 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) sum_small_big_all2 = (sum_small_big_all[:, :] == 2) * 1 sum_intersection_sb = sum_small_big_all2.sum(axis=1).sum() + if sum_intersection_sb > 0: + dis_small_from_bigs_tot = [] for z1 in range(len(textlines_small)): # print(len(textlines_small),'small') @@ -1111,22 +921,27 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) sum_small_big_2 = (sum_small_big[:, :] == 2) * 1 sum_intersection = sum_small_big_2.sum(axis=1).sum() + # print(sum_intersection) + intersections.append(sum_intersection) if len(np.array(intersections)[np.array(intersections) > 0]) == 0: intersections = [] + try: dis_small_from_bigs_tot.append(np.argmax(intersections)) except: dis_small_from_bigs_tot.append(-1) smalls_list = np.array(dis_small_from_bigs_tot)[np.array(dis_small_from_bigs_tot) >= 0] + # index_small_textlines_rest=list( set(indexes_textlines_small)-set(smalls_list) ) textlines_big_with_change = [] textlines_big_with_change_con = [] textlines_small_with_change = [] + for z in list(set(smalls_list)): index_small_textlines = list(np.where(np.array(dis_small_from_bigs_tot) == z)[0]) # print(z,index_small_textlines) @@ -1146,6 +961,7 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) cont, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # print(cont[0],type(cont)) + textlines_big_with_change_con.append(cont) textlines_big_org_form[z] = cont[0] @@ -1156,11 +972,13 @@ def small_textlines_to_parent_adherence2(textlines_con, textline_iamge, num_col) # print(textlines_small_with_change,'textlines_small_with_change') # print(textlines_big) textlines_con_changed.append(textlines_big_org_form) + else: textlines_con_changed.append(textlines_big_org_form) return textlines_con_changed def order_of_regions(textline_mask, contours_main, contours_header, y_ref): + ##plt.imshow(textline_mask) ##plt.show() """ @@ -1170,47 +988,59 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): y_help=np.zeros(len(y)+40) y_help[20:len(y)+20]=y - x=np.arange(len(y)) + x=np.array( range(len(y)) ) + peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + ##plt.imshow(textline_mask[:,:]) ##plt.show() + sigma_gaus=8 + z= gaussian_filter1d(y_help, sigma_gaus) zneg_rev=-y_help+np.max(y_help) + zneg=np.zeros(len(zneg_rev)+40) zneg[20:len(zneg_rev)+20]=zneg_rev zneg= gaussian_filter1d(zneg, sigma_gaus) + peaks, _ = find_peaks(z, height=0) peaks_neg, _ = find_peaks(zneg, height=0) + peaks_neg=peaks_neg-20-20 peaks=peaks-20 """ + textline_sum_along_width = textline_mask.sum(axis=1) y = textline_sum_along_width[:] y_padded = np.zeros(len(y) + 40) y_padded[20 : len(y) + 20] = y - x = np.arange(len(y)) + x = np.array(range(len(y))) peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) sigma_gaus = 8 + z = gaussian_filter1d(y_padded, sigma_gaus) zneg_rev = -y_padded + np.max(y_padded) + zneg = np.zeros(len(zneg_rev) + 40) zneg[20 : len(zneg_rev) + 20] = zneg_rev zneg = gaussian_filter1d(zneg, sigma_gaus) peaks, _ = find_peaks(z, height=0) peaks_neg, _ = find_peaks(zneg, height=0) + peaks_neg = peaks_neg - 20 - 20 peaks = peaks - 20 ##plt.plot(z) ##plt.show() + if contours_main != None: areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] @@ -1236,32 +1066,42 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): # print(cy_main,'mainy') peaks_neg_new = [] + peaks_neg_new.append(0 + y_ref) for iii in range(len(peaks_neg)): peaks_neg_new.append(peaks_neg[iii] + y_ref) + peaks_neg_new.append(textline_mask.shape[0] + y_ref) if len(cy_main) > 0 and np.max(cy_main) > np.max(peaks_neg_new): cy_main = np.array(cy_main) * (np.max(peaks_neg_new) / np.max(cy_main)) - 10 + if contours_main != None: - indexer_main = np.arange(len(contours_main)) + indexer_main = np.array(range(len(contours_main))) + if contours_main != None: len_main = len(contours_main) else: len_main = 0 matrix_of_orders = np.zeros((len(contours_main) + len(contours_header), 5)) - matrix_of_orders[:, 0] = np.arange(len(contours_main) + len(contours_header)) + + matrix_of_orders[:, 0] = np.array(range(len(contours_main) + len(contours_header))) + matrix_of_orders[: len(contours_main), 1] = 1 matrix_of_orders[len(contours_main) :, 1] = 2 + matrix_of_orders[: len(contours_main), 2] = cx_main matrix_of_orders[len(contours_main) :, 2] = cx_header + matrix_of_orders[: len(contours_main), 3] = cy_main matrix_of_orders[len(contours_main) :, 3] = cy_header - matrix_of_orders[: len(contours_main), 4] = np.arange(len(contours_main)) - matrix_of_orders[len(contours_main) :, 4] = np.arange(len(contours_header)) + + matrix_of_orders[: len(contours_main), 4] = np.array(range(len(contours_main))) + matrix_of_orders[len(contours_main) :, 4] = np.array(range(len(contours_header))) # print(peaks_neg_new,'peaks_neg_new') + # print(matrix_of_orders,'matrix_of_orders') # print(peaks_neg_new,np.max(peaks_neg_new)) final_indexers_sorted = [] @@ -1270,20 +1110,24 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): for i in range(len(peaks_neg_new) - 1): top = peaks_neg_new[i] down = peaks_neg_new[i + 1] - indexes_in = matrix_of_orders[:, 0][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] - cxs_in = matrix_of_orders[:, 2][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] - cys_in = matrix_of_orders[:, 3][(matrix_of_orders[:, 3] >= top) & - ((matrix_of_orders[:, 3] < down))] - types_of_text = matrix_of_orders[:, 1][(matrix_of_orders[:, 3] >= top) & - (matrix_of_orders[:, 3] < down)] - index_types_of_text = matrix_of_orders[:, 4][(matrix_of_orders[:, 3] >= top) & - (matrix_of_orders[:, 3] < down)] + + # print(top,down,'topdown') + + indexes_in = matrix_of_orders[:, 0][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + cxs_in = matrix_of_orders[:, 2][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + cys_in = matrix_of_orders[:, 3][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + types_of_text = matrix_of_orders[:, 1][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + index_types_of_text = matrix_of_orders[:, 4][(matrix_of_orders[:, 3] >= top) & ((matrix_of_orders[:, 3] < down))] + + # print(top,down) + # print(cys_in,'cyyyins') + # print(indexes_in,'indexes') sorted_inside = np.argsort(cxs_in) + ind_in_int = indexes_in[sorted_inside] ind_in_type = types_of_text[sorted_inside] ind_ind_type = index_types_of_text[sorted_inside] + for j in range(len(ind_in_int)): final_indexers_sorted.append(int(ind_in_int[j])) final_types.append(int(ind_in_type[j])) @@ -1291,22 +1135,14 @@ def order_of_regions(textline_mask, contours_main, contours_header, y_ref): ##matrix_of_orders[:len_main,4]=final_indexers_sorted[:] - # This fix is applied if the sum of the lengths of contours and contours_h - # does not match final_indexers_sorted. However, this is not the optimal solution.. - if len(cy_main) + len(cy_header) == len(final_index_type): - pass - else: - indexes_missed = set(np.arange(len(cy_main) + len(cy_header))) - set(final_indexers_sorted) - for ind_missed in indexes_missed: - final_indexers_sorted.append(ind_missed) - final_types.append(1) - final_index_type.append(ind_missed) + # print(peaks_neg_new,'peaks') + # print(final_indexers_sorted,'indexsorted') + # print(final_types,'types') + # print(final_index_type,'final_index_type') return final_indexers_sorted, matrix_of_orders, final_types, final_index_type -def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( - img_p_in_ver, img_in_hor,num_col_classifier): - +def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new(img_p_in_ver, img_in_hor,num_col_classifier): #img_p_in_ver = cv2.erode(img_p_in_ver, self.kernel, iterations=2) img_p_in_ver=img_p_in_ver.astype(np.uint8) img_p_in_ver=np.repeat(img_p_in_ver[:, :, np.newaxis], 3, axis=2) @@ -1314,33 +1150,33 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( ret, thresh = cv2.threshold(imgray, 0, 255, 0) contours_lines_ver,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines_ver, _, x_min_main_ver, _, _, _, y_min_main_ver, y_max_main_ver, cx_main_ver = \ - find_features_of_lines(contours_lines_ver) + + slope_lines_ver,dist_x_ver, x_min_main_ver ,x_max_main_ver ,cy_main_ver,slope_lines_org_ver,y_min_main_ver, y_max_main_ver, cx_main_ver=find_features_of_lines(contours_lines_ver) + for i in range(len(x_min_main_ver)): - img_p_in_ver[int(y_min_main_ver[i]): - int(y_min_main_ver[i])+30, - int(cx_main_ver[i])-25: - int(cx_main_ver[i])+25, 0] = 0 - img_p_in_ver[int(y_max_main_ver[i])-30: - int(y_max_main_ver[i]), - int(cx_main_ver[i])-25: - int(cx_main_ver[i])+25, 0] = 0 - + img_p_in_ver[int(y_min_main_ver[i]):int(y_min_main_ver[i])+30,int(cx_main_ver[i])-25:int(cx_main_ver[i])+25,0]=0 + img_p_in_ver[int(y_max_main_ver[i])-30:int(y_max_main_ver[i]),int(cx_main_ver[i])-25:int(cx_main_ver[i])+25,0]=0 + + img_in_hor=img_in_hor.astype(np.uint8) img_in_hor=np.repeat(img_in_hor[:, :, np.newaxis], 3, axis=2) imgray = cv2.cvtColor(img_in_hor, cv2.COLOR_BGR2GRAY) ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_lines_hor,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - slope_lines_hor, dist_x_hor, x_min_main_hor, x_max_main_hor, cy_main_hor, _, _, _, _ = \ - find_features_of_lines(contours_lines_hor) + + slope_lines_hor,dist_x_hor, x_min_main_hor ,x_max_main_hor ,cy_main_hor,slope_lines_org_hor,y_min_main_hor, y_max_main_hor, cx_main_hor=find_features_of_lines(contours_lines_hor) + + x_width_smaller_than_acolumn_width=img_in_hor.shape[1]/float(num_col_classifier+1.) - + len_lines_bigger_than_x_width_smaller_than_acolumn_width=len( dist_x_hor[dist_x_hor>=x_width_smaller_than_acolumn_width] ) - len_lines_bigger_than_x_width_smaller_than_acolumn_width_per_column=int(len_lines_bigger_than_x_width_smaller_than_acolumn_width / - float(num_col_classifier)) - if len_lines_bigger_than_x_width_smaller_than_acolumn_width_per_column < 10: - args_hor=np.arange(len(slope_lines_hor)) + + len_lines_bigger_than_x_width_smaller_than_acolumn_width_per_column=int( len_lines_bigger_than_x_width_smaller_than_acolumn_width/float(num_col_classifier) ) + + + if len_lines_bigger_than_x_width_smaller_than_acolumn_width_per_column<10: + args_hor=np.array( range(len(slope_lines_hor) )) all_args_uniq=contours_in_same_horizon(cy_main_hor) #print(all_args_uniq,'all_args_uniq') if len(all_args_uniq)>0: @@ -1353,32 +1189,26 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( some_cy=cy_main_hor[all_args_uniq[dd]] some_x_min=x_min_main_hor[all_args_uniq[dd]] some_x_max=x_max_main_hor[all_args_uniq[dd]] - + #img_in=np.zeros(separators_closeup_n[:,:,2].shape) #print(img_p_in_ver.shape[1],some_x_max-some_x_min,'xdiff') diff_x_some=some_x_max-some_x_min for jv in range(len(some_args)): - img_p_in=cv2.fillPoly(img_in_hor, pts=[contours_lines_hor[some_args[jv]]], color=(1,1,1)) + + img_p_in=cv2.fillPoly(img_in_hor, pts =[contours_lines_hor[some_args[jv]]], color=(1,1,1)) + if any(i_diff>(img_p_in_ver.shape[1]/float(3.3)) for i_diff in diff_x_some): - img_p_in[int(np.mean(some_cy))-5: - int(np.mean(some_cy))+5, - int(np.min(some_x_min)): - int(np.max(some_x_max)) ]=1 + img_p_in[int(np.mean(some_cy))-5:int(np.mean(some_cy))+5, int(np.min(some_x_min)):int(np.max(some_x_max)) ]=1 + sum_dis=dist_x_hor[some_args].sum() diff_max_min_uniques=np.max(x_max_main_hor[some_args])-np.min(x_min_main_hor[some_args]) - - if (diff_max_min_uniques > sum_dis and - sum_dis / float(diff_max_min_uniques) > 0.85 and - diff_max_min_uniques / float(img_p_in_ver.shape[1]) > 0.85 and - np.std(dist_x_hor[some_args]) < 0.55 * np.mean(dist_x_hor[some_args])): - # print(dist_x_hor[some_args], - # dist_x_hor[some_args].sum(), - # np.min(x_min_main_hor[some_args]), - # np.max(x_max_main_hor[some_args]),'jalibdi') - # print(np.mean( dist_x_hor[some_args] ), - # np.std( dist_x_hor[some_args] ), - # np.var( dist_x_hor[some_args] ),'jalibdiha') + + + if diff_max_min_uniques>sum_dis and ( (sum_dis/float(diff_max_min_uniques) ) >0.85 ) and ( (diff_max_min_uniques/float(img_p_in_ver.shape[1]))>0.85 ) and np.std( dist_x_hor[some_args] )<(0.55*np.mean( dist_x_hor[some_args] )): + #print(dist_x_hor[some_args],dist_x_hor[some_args].sum(),np.min(x_min_main_hor[some_args]) ,np.max(x_max_main_hor[some_args]),'jalibdi') + #print(np.mean( dist_x_hor[some_args] ),np.std( dist_x_hor[some_args] ),np.var( dist_x_hor[some_args] ),'jalibdiha') special_separators.append(np.mean(cy_main_hor[some_args])) + else: img_p_in=img_in_hor special_separators=[] @@ -1386,22 +1216,29 @@ def combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( img_p_in=img_in_hor special_separators=[] + img_p_in_ver[:,:,0][img_p_in_ver[:,:,0]==255]=1 sep_ver_hor=img_p_in+img_p_in_ver + + sep_ver_hor_cross=(sep_ver_hor[:,:,0]==2)*1 + sep_ver_hor_cross=np.repeat(sep_ver_hor_cross[:, :, np.newaxis], 3, axis=2) sep_ver_hor_cross=sep_ver_hor_cross.astype(np.uint8) imgray = cv2.cvtColor(sep_ver_hor_cross, cv2.COLOR_BGR2GRAY) ret, thresh = cv2.threshold(imgray, 0, 255, 0) contours_cross,_=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) + cx_cross,cy_cross ,_ , _, _ ,_,_=find_new_features_of_contours(contours_cross) + for ii in range(len(cx_cross)): img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])+5:int(cx_cross[ii])+40,0]=0 img_p_in[int(cy_cross[ii])-30:int(cy_cross[ii])+30,int(cx_cross[ii])-40:int(cx_cross[ii])-4,0]=0 + else: img_p_in=np.copy(img_in_hor) special_separators=[] - return img_p_in[:,:,0], special_separators + return img_p_in[:,:,0],special_separators def return_points_with_boundies(peaks_neg_fin, first_point, last_point): peaks_neg_tot = [] @@ -1411,52 +1248,102 @@ def return_points_with_boundies(peaks_neg_fin, first_point, last_point): peaks_neg_tot.append(last_point) return peaks_neg_tot -def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, pixel_lines, contours_h=None): - t_ins_c0 = time.time() +def find_number_of_columns_in_document(region_pre_p, num_col_classifier, pixel_lines, contours_h=None): + separators_closeup=( (region_pre_p[:,:,:]==pixel_lines))*1 + separators_closeup[0:110,:,:]=0 separators_closeup[separators_closeup.shape[0]-150:,:,:]=0 - + kernel = np.ones((5,5),np.uint8) + separators_closeup=separators_closeup.astype(np.uint8) separators_closeup = cv2.dilate(separators_closeup,kernel,iterations = 1) separators_closeup = cv2.erode(separators_closeup,kernel,iterations = 1) + separators_closeup_new=np.zeros((separators_closeup.shape[0] ,separators_closeup.shape[1] )) + + + + ##_,separators_closeup_n=self.combine_hor_lines_and_delete_cross_points_and_get_lines_features_back(region_pre_p[:,:,0]) separators_closeup_n=np.copy(separators_closeup) + separators_closeup_n=separators_closeup_n.astype(np.uint8) - + ##plt.imshow(separators_closeup_n[:,:,0]) + ##plt.show() + separators_closeup_n_binary=np.zeros(( separators_closeup_n.shape[0],separators_closeup_n.shape[1]) ) separators_closeup_n_binary[:,:]=separators_closeup_n[:,:,0] + separators_closeup_n_binary[:,:][separators_closeup_n_binary[:,:]!=0]=1 - + #separators_closeup_n_binary[:,:][separators_closeup_n_binary[:,:]==0]=255 + #separators_closeup_n_binary[:,:][separators_closeup_n_binary[:,:]==-255]=0 + + + #separators_closeup_n_binary=(separators_closeup_n_binary[:,:]==2)*1 + + #gray = cv2.cvtColor(separators_closeup_n, cv2.COLOR_BGR2GRAY) + + ### + + #print(separators_closeup_n_binary.shape) gray_early=np.repeat(separators_closeup_n_binary[:, :, np.newaxis], 3, axis=2) gray_early=gray_early.astype(np.uint8) + + #print(gray_early.shape,'burda') imgray_e = cv2.cvtColor(gray_early, cv2.COLOR_BGR2GRAY) + #print('burda2') ret_e, thresh_e = cv2.threshold(imgray_e, 0, 255, 0) - + + #print('burda3') contours_line_e,hierarchy_e=cv2.findContours(thresh_e,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - _, dist_xe, _, _, _, _, y_min_main, y_max_main, _ = \ - find_features_of_lines(contours_line_e) - dist_ye = y_max_main - y_min_main - args_e=np.arange(len(contours_line_e)) - args_hor_e=args_e[(dist_ye<=50) & - (dist_xe>=3*dist_ye)] + + #slope_lines_e,dist_x_e, x_min_main_e ,x_max_main_e ,cy_main_e,slope_lines_org_e,y_min_main_e, y_max_main_e, cx_main_e=self.find_features_of_lines(contours_line_e) + + slope_linese,dist_xe, x_min_maine ,x_max_maine ,cy_maine,slope_lines_orge,y_min_maine, y_max_maine, cx_maine=find_features_of_lines(contours_line_e) + + dist_ye=y_max_maine-y_min_maine + #print(y_max_maine-y_min_maine,'y') + #print(dist_xe,'x') + + + args_e=np.array(range(len(contours_line_e))) + args_hor_e=args_e[(dist_ye<=50) & (dist_xe>=3*dist_ye)] + + #print(args_hor_e,'jidi',len(args_hor_e),'jilva') + cnts_hor_e=[] for ce in args_hor_e: cnts_hor_e.append(contours_line_e[ce]) + #print(len(slope_linese),'lieee') + figs_e=np.zeros(thresh_e.shape) figs_e=cv2.fillPoly(figs_e,pts=cnts_hor_e,color=(1,1,1)) - - separators_closeup_n_binary=cv2.fillPoly(separators_closeup_n_binary, pts=cnts_hor_e, color=(0,0,0)) + + #plt.imshow(figs_e) + #plt.show() + + ### + + separators_closeup_n_binary=cv2.fillPoly(separators_closeup_n_binary,pts=cnts_hor_e,color=(0,0,0)) + gray = cv2.bitwise_not(separators_closeup_n_binary) gray=gray.astype(np.uint8) + + #plt.imshow(gray) + #plt.show() + + bw = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, \ - cv2.THRESH_BINARY, 15, -2) + cv2.THRESH_BINARY, 15, -2) + ##plt.imshow(bw[:,:]) + ##plt.show() + horizontal = np.copy(bw) vertical = np.copy(bw) - + cols = horizontal.shape[1] horizontal_size = cols // 30 # Create structure element for extracting horizontal lines through morphology operations @@ -1466,10 +1353,22 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, horizontal = cv2.dilate(horizontal, horizontalStructure) kernel = np.ones((5,5),np.uint8) + + horizontal = cv2.dilate(horizontal,kernel,iterations = 2) horizontal = cv2.erode(horizontal,kernel,iterations = 2) - horizontal = cv2.fillPoly(horizontal, pts=cnts_hor_e, color=(255,255,255)) - + + + ### + #print(np.unique(horizontal),'uni') + horizontal=cv2.fillPoly(horizontal,pts=cnts_hor_e,color=(255,255,255)) + ### + + + + #plt.imshow(horizontal) + #plt.show() + rows = vertical.shape[0] verticalsize = rows // 30 # Create structure element for extracting vertical lines through morphology operations @@ -1477,26 +1376,38 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, # Apply morphology operations vertical = cv2.erode(vertical, verticalStructure) vertical = cv2.dilate(vertical, verticalStructure) + vertical = cv2.dilate(vertical,kernel,iterations = 1) + # Show extracted vertical lines - horizontal, special_separators = \ - combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new( - vertical, horizontal, num_col_classifier) - + horizontal,special_separators=combine_hor_lines_and_delete_cross_points_and_get_lines_features_back_new(vertical,horizontal,num_col_classifier) + + + #plt.imshow(horizontal) + #plt.show() + #print(vertical.shape,np.unique(vertical),'verticalvertical') separators_closeup_new[:,:][vertical[:,:]!=0]=1 separators_closeup_new[:,:][horizontal[:,:]!=0]=1 - + + ##plt.imshow(separators_closeup_new) + ##plt.show() + ##separators_closeup_n vertical=np.repeat(vertical[:, :, np.newaxis], 3, axis=2) vertical=vertical.astype(np.uint8) + + ##plt.plot(vertical[:,:,0].sum(axis=0)) + ##plt.show() + + #plt.plot(vertical[:,:,0].sum(axis=1)) + #plt.show() imgray = cv2.cvtColor(vertical, cv2.COLOR_BGR2GRAY) ret, thresh = cv2.threshold(imgray, 0, 255, 0) - + contours_line_vers,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = \ - find_features_of_lines(contours_line_vers) - - args=np.arange(len(slope_lines)) + slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=find_features_of_lines(contours_line_vers) + #print(slope_lines,'vertical') + args=np.array( range(len(slope_lines) )) args_ver=args[slope_lines==1] dist_x_ver=dist_x[slope_lines==1] y_min_main_ver=y_min_main[slope_lines==1] @@ -1506,20 +1417,25 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, cx_main_ver=cx_main[slope_lines==1] dist_y_ver=y_max_main_ver-y_min_main_ver len_y=separators_closeup.shape[0]/3.0 - + + + #plt.imshow(horizontal) + #plt.show() + horizontal=np.repeat(horizontal[:, :, np.newaxis], 3, axis=2) horizontal=horizontal.astype(np.uint8) imgray = cv2.cvtColor(horizontal, cv2.COLOR_BGR2GRAY) ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_line_hors,hierarchy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - slope_lines, dist_x, x_min_main, x_max_main, cy_main, slope_lines_org, y_min_main, y_max_main, cx_main = \ - find_features_of_lines(contours_line_hors) - + slope_lines,dist_x, x_min_main ,x_max_main ,cy_main,slope_lines_org,y_min_main, y_max_main, cx_main=find_features_of_lines(contours_line_hors) + slope_lines_org_hor=slope_lines_org[slope_lines==0] - args=np.arange(len(slope_lines)) + args=np.array( range(len(slope_lines) )) len_x=separators_closeup.shape[1]/5.0 - dist_y=np.abs(y_max_main-y_min_main) + dist_y=np.abs(y_max_main-y_min_main) + args_hor=args[slope_lines==0] dist_x_hor=dist_x[slope_lines==0] y_min_main_hor=y_min_main[slope_lines==0] @@ -1536,351 +1452,413 @@ def find_number_of_columns_in_document(region_pre_p, num_col_classifier, tables, y_min_main_hor=y_min_main_hor[dist_x_hor>=len_x/2.0] y_max_main_hor=y_max_main_hor[dist_x_hor>=len_x/2.0] dist_y_hor=dist_y_hor[dist_x_hor>=len_x/2.0] + slope_lines_org_hor=slope_lines_org_hor[dist_x_hor>=len_x/2.0] dist_x_hor=dist_x_hor[dist_x_hor>=len_x/2.0] - + + matrix_of_lines_ch=np.zeros((len(cy_main_hor)+len(cx_main_ver),10)) + matrix_of_lines_ch[:len(cy_main_hor),0]=args_hor matrix_of_lines_ch[len(cy_main_hor):,0]=args_ver + + matrix_of_lines_ch[len(cy_main_hor):,1]=cx_main_ver + matrix_of_lines_ch[:len(cy_main_hor),2]=x_min_main_hor+50#x_min_main_hor+150 matrix_of_lines_ch[len(cy_main_hor):,2]=x_min_main_ver + matrix_of_lines_ch[:len(cy_main_hor),3]=x_max_main_hor-50#x_max_main_hor-150 matrix_of_lines_ch[len(cy_main_hor):,3]=x_max_main_ver + matrix_of_lines_ch[:len(cy_main_hor),4]=dist_x_hor matrix_of_lines_ch[len(cy_main_hor):,4]=dist_x_ver + matrix_of_lines_ch[:len(cy_main_hor),5]=cy_main_hor + + matrix_of_lines_ch[:len(cy_main_hor),6]=y_min_main_hor matrix_of_lines_ch[len(cy_main_hor):,6]=y_min_main_ver + matrix_of_lines_ch[:len(cy_main_hor),7]=y_max_main_hor matrix_of_lines_ch[len(cy_main_hor):,7]=y_max_main_ver + matrix_of_lines_ch[:len(cy_main_hor),8]=dist_y_hor matrix_of_lines_ch[len(cy_main_hor):,8]=dist_y_ver - matrix_of_lines_ch[len(cy_main_hor):,9]=1 + + matrix_of_lines_ch[len(cy_main_hor):,9]=1 + + + if contours_h is not None: - _, dist_x_head, x_min_main_head, x_max_main_head, cy_main_head, _, y_min_main_head, y_max_main_head, _ = \ - find_features_of_lines(contours_h) + slope_lines_head,dist_x_head, x_min_main_head ,x_max_main_head ,cy_main_head,slope_lines_org_head,y_min_main_head, y_max_main_head, cx_main_head=find_features_of_lines(contours_h) matrix_l_n=np.zeros((matrix_of_lines_ch.shape[0]+len(cy_main_head),matrix_of_lines_ch.shape[1])) matrix_l_n[:matrix_of_lines_ch.shape[0],:]=np.copy(matrix_of_lines_ch[:,:]) - args_head=np.arange(len(cy_main_head)) + len(cy_main_hor) - + args_head=np.array(range(len(cy_main_head)))+len(cy_main_hor) + matrix_l_n[matrix_of_lines_ch.shape[0]:,0]=args_head matrix_l_n[matrix_of_lines_ch.shape[0]:,2]=x_min_main_head+30 matrix_l_n[matrix_of_lines_ch.shape[0]:,3]=x_max_main_head-30 + matrix_l_n[matrix_of_lines_ch.shape[0]:,4]=dist_x_head + matrix_l_n[matrix_of_lines_ch.shape[0]:,5]=y_min_main_head-3-8 matrix_l_n[matrix_of_lines_ch.shape[0]:,6]=y_min_main_head-5-8 matrix_l_n[matrix_of_lines_ch.shape[0]:,7]=y_max_main_head#y_min_main_head+1-8 matrix_l_n[matrix_of_lines_ch.shape[0]:,8]=4 + matrix_of_lines_ch=np.copy(matrix_l_n) - - cy_main_splitters=cy_main_hor[(x_min_main_hor<=.16*region_pre_p.shape[1]) & - (x_max_main_hor>=.84*region_pre_p.shape[1])] + + + cy_main_splitters=cy_main_hor[ (x_min_main_hor<=.16*region_pre_p.shape[1]) & (x_max_main_hor>=.84*region_pre_p.shape[1] )] + cy_main_splitters=np.array( list(cy_main_splitters)+list(special_separators)) + if contours_h is not None: try: - cy_main_splitters_head=cy_main_head[(x_min_main_head<=.16*region_pre_p.shape[1]) & - (x_max_main_head>=.84*region_pre_p.shape[1])] + cy_main_splitters_head=cy_main_head[ (x_min_main_head<=.16*region_pre_p.shape[1]) & (x_max_main_head>=.84*region_pre_p.shape[1] )] cy_main_splitters=np.array( list(cy_main_splitters)+list(cy_main_splitters_head)) except: pass args_cy_splitter=np.argsort(cy_main_splitters) + cy_main_splitters_sort=cy_main_splitters[args_cy_splitter] - + splitter_y_new=[] splitter_y_new.append(0) for i in range(len(cy_main_splitters_sort)): - splitter_y_new.append( cy_main_splitters_sort[i] ) + splitter_y_new.append( cy_main_splitters_sort[i] ) + splitter_y_new.append(region_pre_p.shape[0]) + splitter_y_new_diff=np.diff(splitter_y_new)/float(region_pre_p.shape[0])*100 - - args_big_parts=np.arange(len(splitter_y_new_diff))[ splitter_y_new_diff>22 ] - + + args_big_parts=np.array(range(len(splitter_y_new_diff))) [ splitter_y_new_diff>22 ] + + + regions_without_separators=return_regions_without_separators(region_pre_p) + + length_y_threshold=regions_without_separators.shape[0]/4.0 - + num_col_fin=0 peaks_neg_fin_fin=[] + for itiles in args_big_parts: - regions_without_separators_tile=regions_without_separators[int(splitter_y_new[itiles]): - int(splitter_y_new[itiles+1]),:,0] + + + regions_without_separators_tile=regions_without_separators[int(splitter_y_new[itiles]):int(splitter_y_new[itiles+1]),:,0] + #image_page_background_zero_tile=image_page_background_zero[int(splitter_y_new[itiles]):int(splitter_y_new[itiles+1]),:] + + #print(regions_without_separators_tile.shape) + ##plt.imshow(regions_without_separators_tile) + ##plt.show() + + #num_col, peaks_neg_fin=self.find_num_col(regions_without_separators_tile,multiplier=6.0) + + #regions_without_separators_tile=cv2.erode(regions_without_separators_tile,kernel,iterations = 3) + # try: - num_col, peaks_neg_fin = find_num_col(regions_without_separators_tile, - num_col_classifier, tables, multiplier=7.0) + num_col, peaks_neg_fin = find_num_col(regions_without_separators_tile,multiplier=7.0) except: num_col = 0 peaks_neg_fin = [] + if num_col>num_col_fin: num_col_fin=num_col peaks_neg_fin_fin=peaks_neg_fin + if len(args_big_parts)==1 and (len(peaks_neg_fin_fin)+1)=500] peaks_neg_fin=peaks_neg_fin[peaks_neg_fin<=(vertical.shape[1]-500)] peaks_neg_fin_fin=peaks_neg_fin[:] - + + #print(peaks_neg_fin_fin,'peaks_neg_fin_fintaza') + + return num_col_fin, peaks_neg_fin_fin,matrix_of_lines_ch,splitter_y_new,separators_closeup_n + -def return_boxes_of_images_by_order_of_reading_new( - splitter_y_new, regions_without_separators, - matrix_of_lines_ch, - num_col_classifier, erosion_hurts, tables, right2left_readingorder): - - if right2left_readingorder: - regions_without_separators = cv2.flip(regions_without_separators,1) +def return_boxes_of_images_by_order_of_reading_new(splitter_y_new, regions_without_separators, matrix_of_lines_ch, num_col_classifier, erosion_hurts): boxes=[] - peaks_neg_tot_tables = [] + + for i in range(len(splitter_y_new)-1): #print(splitter_y_new[i],splitter_y_new[i+1]) - matrix_new = matrix_of_lines_ch[:,:][(matrix_of_lines_ch[:,6]> splitter_y_new[i] ) & - (matrix_of_lines_ch[:,7]< splitter_y_new[i+1] )] + matrix_new=matrix_of_lines_ch[:,:][ (matrix_of_lines_ch[:,6]> splitter_y_new[i] ) & (matrix_of_lines_ch[:,7]< splitter_y_new[i+1] ) ] #print(len( matrix_new[:,9][matrix_new[:,9]==1] )) + #print(matrix_new[:,8][matrix_new[:,9]==1],'gaddaaa') + # check to see is there any vertical separator to find holes. - #if (len(matrix_new[:,9][matrix_new[:,9]==1]) > 0 and - # np.max(matrix_new[:,8][matrix_new[:,9]==1]) >= - # 0.1 * (np.abs(splitter_y_new[i+1]-splitter_y_new[i]))): - if True: + if 1>0:#len( matrix_new[:,9][matrix_new[:,9]==1] )>0 and np.max(matrix_new[:,8][matrix_new[:,9]==1])>=0.1*(np.abs(splitter_y_new[i+1]-splitter_y_new[i] )): + try: if erosion_hurts: - num_col, peaks_neg_fin = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], - num_col_classifier, tables, multiplier=6.) + num_col, peaks_neg_fin=find_num_col(regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:],multiplier=6.) else: - num_col, peaks_neg_fin = find_num_col( - regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], - num_col_classifier, tables, multiplier=7.) + num_col, peaks_neg_fin=find_num_col(regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:],multiplier=7.) except: peaks_neg_fin=[] - num_col = 0 + + try: peaks_neg_fin_org=np.copy(peaks_neg_fin) - if (len(peaks_neg_fin)+1)=len(peaks_neg_fin2): peaks_neg_fin=list(np.copy(peaks_neg_fin1)) else: peaks_neg_fin=list(np.copy(peaks_neg_fin2)) + + + peaks_neg_fin=list(np.array(peaks_neg_fin)+peaks_neg_fin_early[i_n]) - + if i_n!=(len(peaks_neg_fin_early)-2): peaks_neg_fin_rev.append(peaks_neg_fin_early[i_n+1]) #print(peaks_neg_fin,'peaks_neg_fin') peaks_neg_fin_rev=peaks_neg_fin_rev+peaks_neg_fin - if len(peaks_neg_fin_rev)>=len(peaks_neg_fin_org): + + + + + if len(peaks_neg_fin_rev)>=len(peaks_neg_fin_org): peaks_neg_fin=list(np.sort(peaks_neg_fin_rev)) num_col=len(peaks_neg_fin) else: peaks_neg_fin=list(np.copy(peaks_neg_fin_org)) num_col=len(peaks_neg_fin) - + #print(peaks_neg_fin,'peaks_neg_fin') except: pass - #num_col, peaks_neg_fin = find_num_col( - # regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:], - # multiplier=7.0) + #num_col, peaks_neg_fin=find_num_col(regions_without_separators[int(splitter_y_new[i]):int(splitter_y_new[i+1]),:],multiplier=7.0) x_min_hor_some=matrix_new[:,2][ (matrix_new[:,9]==0) ] x_max_hor_some=matrix_new[:,3][ (matrix_new[:,9]==0) ] cy_hor_some=matrix_new[:,5][ (matrix_new[:,9]==0) ] cy_hor_diff=matrix_new[:,7][ (matrix_new[:,9]==0) ] arg_org_hor_some=matrix_new[:,0][ (matrix_new[:,9]==0) ] - - if right2left_readingorder: - x_max_hor_some_new = regions_without_separators.shape[1] - x_min_hor_some - x_min_hor_some_new = regions_without_separators.shape[1] - x_max_hor_some - x_min_hor_some =list(np.copy(x_min_hor_some_new)) - x_max_hor_some =list(np.copy(x_max_hor_some_new)) + + + + peaks_neg_tot=return_points_with_boundies(peaks_neg_fin,0, regions_without_separators[:,:].shape[1]) - peaks_neg_tot_tables.append(peaks_neg_tot) + + reading_order_type,x_starting,x_ending,y_type_2,y_diff_type_2,y_lines_without_mother,x_start_without_mother,x_end_without_mother,there_is_sep_with_child,y_lines_with_child_without_mother,x_start_with_child_without_mother,x_end_with_child_without_mother=return_x_start_end_mothers_childs_and_type_of_reading_order(x_min_hor_some,x_max_hor_some,cy_hor_some,peaks_neg_tot,cy_hor_diff) + - reading_order_type, x_starting, x_ending, y_type_2, y_diff_type_2, \ - y_lines_without_mother, x_start_without_mother, x_end_without_mother, there_is_sep_with_child, \ - y_lines_with_child_without_mother, x_start_with_child_without_mother, x_end_with_child_without_mother, \ - new_main_sep_y = return_x_start_end_mothers_childs_and_type_of_reading_order( - x_min_hor_some, x_max_hor_some, cy_hor_some, peaks_neg_tot, cy_hor_diff) - x_starting = np.array(x_starting) - x_ending = np.array(x_ending) - y_type_2 = np.array(y_type_2) - y_diff_type_2 = np.array(y_diff_type_2) + + if (reading_order_type==1) or (reading_order_type==0 and (len(y_lines_without_mother)>=2 or there_is_sep_with_child==1)): - if ((reading_order_type==1) or - (reading_order_type==0 and - (len(y_lines_without_mother)>=2 or there_is_sep_with_child==1))): + try: y_grenze=int(splitter_y_new[i])+300 + + + #check if there is a big separator in this y_mains_sep_ohne_grenzen - - args_early_ys=np.arange(len(y_type_2)) + + args_early_ys=np.array(range(len(y_type_2))) + #print(args_early_ys,'args_early_ys') #print(int(splitter_y_new[i]),int(splitter_y_new[i+1])) - - x_starting_up = x_starting[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - x_ending_up = x_ending[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - y_type_2_up = y_type_2[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - y_diff_type_2_up = y_diff_type_2[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - args_up = args_early_ys[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - if len(y_type_2_up) > 0: - y_main_separator_up = y_type_2_up [(x_starting_up==0) & - (x_ending_up==(len(peaks_neg_tot)-1) )] - y_diff_main_separator_up = y_diff_type_2_up[(x_starting_up==0) & - (x_ending_up==(len(peaks_neg_tot)-1) )] - args_main_to_deleted = args_up[(x_starting_up==0) & - (x_ending_up==(len(peaks_neg_tot)-1) )] + + y_type_2_up=np.array(y_type_2)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + x_starting_up=np.array(x_starting)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + x_ending_up=np.array(x_ending)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + y_diff_type_2_up=np.array(y_diff_type_2)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + args_up=args_early_ys[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + + + + if len(y_type_2_up)>0: + y_main_separator_up=y_type_2_up[(x_starting_up==0) & (x_ending_up==(len(peaks_neg_tot)-1) )] + y_diff_main_separator_up=y_diff_type_2_up[(x_starting_up==0) & (x_ending_up==(len(peaks_neg_tot)-1) )] + args_main_to_deleted=args_up[(x_starting_up==0) & (x_ending_up==(len(peaks_neg_tot)-1) )] #print(y_main_separator_up,y_diff_main_separator_up,args_main_to_deleted,'fffffjammmm') - if len(y_diff_main_separator_up) > 0: - args_to_be_kept = np.array(list( set(args_early_ys) - set(args_main_to_deleted) )) + + if len(y_diff_main_separator_up)>0: + args_to_be_kept=np.array( list( set(args_early_ys)-set(args_main_to_deleted) ) ) #print(args_to_be_kept,'args_to_be_kept') - boxes.append([0, peaks_neg_tot[len(peaks_neg_tot)-1], - int(splitter_y_new[i]), int( np.max(y_diff_main_separator_up))]) + boxes.append([0,peaks_neg_tot[len(peaks_neg_tot)-1],int(splitter_y_new[i]),int( np.max(y_diff_main_separator_up))]) splitter_y_new[i]=[ np.max(y_diff_main_separator_up) ][0] - + #print(splitter_y_new[i],'splitter_y_new[i]') - y_type_2 = y_type_2[args_to_be_kept] - x_starting = x_starting[args_to_be_kept] - x_ending = x_ending[args_to_be_kept] - y_diff_type_2 = y_diff_type_2[args_to_be_kept] - + y_type_2=np.array(y_type_2)[args_to_be_kept] + x_starting=np.array(x_starting)[args_to_be_kept] + x_ending=np.array(x_ending)[args_to_be_kept] + y_diff_type_2=np.array(y_diff_type_2)[args_to_be_kept] + #print('galdiha') y_grenze=int(splitter_y_new[i])+200 - args_early_ys2=np.arange(len(y_type_2)) - y_type_2_up=y_type_2[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - x_starting_up=x_starting[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - x_ending_up=x_ending[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - y_diff_type_2_up=y_diff_type_2[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] - args_up2=args_early_ys2[(y_type_2 > int(splitter_y_new[i])) & - (y_type_2 <= y_grenze)] + + + args_early_ys2=np.array(range(len(y_type_2))) + y_type_2_up=np.array(y_type_2)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + x_starting_up=np.array(x_starting)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + x_ending_up=np.array(x_ending)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + y_diff_type_2_up=np.array(y_diff_type_2)[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + args_up2=args_early_ys2[( np.array(y_type_2)>int(splitter_y_new[i]) ) & (np.array(y_type_2)<=y_grenze)] + + #print(y_type_2_up,x_starting_up,x_ending_up,'didid') - nodes_in = [] + + nodes_in=[] for ij in range(len(x_starting_up)): - nodes_in = nodes_in + list(range(x_starting_up[ij], - x_ending_up[ij])) - nodes_in = np.unique(nodes_in) - #print(nodes_in,'nodes_in') - - if set(nodes_in)==set(range(len(peaks_neg_tot)-1)): + nodes_in=nodes_in+list(np.array(range(x_starting_up[ij],x_ending_up[ij]))) + + #print(np.unique(nodes_in),'nodes_in') + + if set(np.unique(nodes_in))==set(np.array(range(len(peaks_neg_tot)-1)) ): pass - elif set(nodes_in)==set(range(1, len(peaks_neg_tot)-1)): + elif set( np.unique(nodes_in) )==set( np.array(range(1,len(peaks_neg_tot)-1)) ): pass else: #print('burdaydikh') - args_to_be_kept2=np.array(list( set(args_early_ys2)-set(args_up2) )) - + args_to_be_kept2=np.array( list( set(args_early_ys2)-set(args_up2) ) ) + if len(args_to_be_kept2)>0: - y_type_2 = y_type_2[args_to_be_kept2] - x_starting = x_starting[args_to_be_kept2] - x_ending = x_ending[args_to_be_kept2] - y_diff_type_2 = y_diff_type_2[args_to_be_kept2] + y_type_2=np.array(y_type_2)[args_to_be_kept2] + x_starting=np.array(x_starting)[args_to_be_kept2] + x_ending=np.array(x_ending)[args_to_be_kept2] + y_diff_type_2=np.array(y_diff_type_2)[args_to_be_kept2] else: pass + #print('burdaydikh2') + + + elif len(y_diff_main_separator_up)==0: - nodes_in = [] + nodes_in=[] for ij in range(len(x_starting_up)): - nodes_in = nodes_in + list(range(x_starting_up[ij], - x_ending_up[ij])) - nodes_in = np.unique(nodes_in) - #print(nodes_in,'nodes_in2') + nodes_in=nodes_in+list(np.array(range(x_starting_up[ij],x_ending_up[ij]))) + + #print(np.unique(nodes_in),'nodes_in2') #print(np.array(range(len(peaks_neg_tot)-1)),'np.array(range(len(peaks_neg_tot)-1))') - - if set(nodes_in)==set(range(len(peaks_neg_tot)-1)): + + + + if set(np.unique(nodes_in))==set(np.array(range(len(peaks_neg_tot)-1)) ): pass - elif set(nodes_in)==set(range(1,len(peaks_neg_tot)-1)): + elif set(np.unique(nodes_in) )==set( np.array(range(1,len(peaks_neg_tot)-1)) ): pass else: #print('burdaydikh') #print(args_early_ys,'args_early_ys') #print(args_up,'args_up') - args_to_be_kept2=np.array(list( set(args_early_ys) - set(args_up) )) - + args_to_be_kept2=np.array( list( set(args_early_ys)-set(args_up) ) ) + #print(args_to_be_kept2,'args_to_be_kept2') + #print(len(y_type_2),len(x_starting),len(x_ending),len(y_diff_type_2)) + if len(args_to_be_kept2)>0: - y_type_2 = y_type_2[args_to_be_kept2] - x_starting = x_starting[args_to_be_kept2] - x_ending = x_ending[args_to_be_kept2] - y_diff_type_2 = y_diff_type_2[args_to_be_kept2] + y_type_2=np.array(y_type_2)[args_to_be_kept2] + x_starting=np.array(x_starting)[args_to_be_kept2] + x_ending=np.array(x_ending)[args_to_be_kept2] + y_diff_type_2=np.array(y_diff_type_2)[args_to_be_kept2] else: pass + #print('burdaydikh2') - + + + + + + + x_starting=np.array(x_starting) + x_ending=np.array(x_ending) + y_type_2=np.array(y_type_2) + y_diff_type_2_up=np.array(y_diff_type_2_up) + #int(splitter_y_new[i]) + y_lines_by_order=[] x_start_by_order=[] x_end_by_order=[] + if (len(x_end_with_child_without_mother)==0 and reading_order_type==0) or reading_order_type==1: + + if reading_order_type==1: y_lines_by_order.append(int(splitter_y_new[i])) x_start_by_order.append(0) x_end_by_order.append(len(peaks_neg_tot)-2) else: #print(x_start_without_mother,x_end_without_mother,peaks_neg_tot,'dodo') - columns_covered_by_mothers = [] + + columns_covered_by_mothers=[] + for dj in range(len(x_start_without_mother)): - columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_start_without_mother[dj], - x_end_without_mother[dj])) - columns_covered_by_mothers = list(set(columns_covered_by_mothers)) + columns_covered_by_mothers=columns_covered_by_mothers+list(np.array(range(x_start_without_mother[dj],x_end_without_mother[dj])) ) + columns_covered_by_mothers=list(set(columns_covered_by_mothers)) + + all_columns=np.array(range(len(peaks_neg_tot)-1)) + + columns_not_covered=list( set(all_columns)-set(columns_covered_by_mothers) ) + + + y_type_2=list(y_type_2) + x_starting=list(x_starting) + x_ending=list(x_ending) + + for lj in columns_not_covered: + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(lj) + x_ending.append(lj+1) + ##y_lines_by_order.append(int(splitter_y_new[i])) + ##x_start_by_order.append(0) + for lk in range(len(x_start_without_mother)): + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(x_start_without_mother[lk]) + x_ending.append(x_end_without_mother[lk]) + + + y_type_2=np.array(y_type_2) + x_starting=np.array(x_starting) + x_ending=np.array(x_ending) + + + - all_columns=np.arange(len(peaks_neg_tot)-1) - columns_not_covered=list(set(all_columns) - set(columns_covered_by_mothers)) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * (len(columns_not_covered) + len(x_start_without_mother))) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) - ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) - x_starting = np.append(x_starting, x_start_without_mother) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) - x_ending = np.append(x_ending, x_end_without_mother) - - ind_args=np.arange(len(y_type_2)) + ind_args=np.array(range(len(y_type_2))) #ind_args=np.array(ind_args) #print(ind_args,'ind_args') for column in range(len(peaks_neg_tot)-1): @@ -1904,115 +1882,158 @@ def return_boxes_of_images_by_order_of_reading_new( y_lines_by_order.append(y_col_sort[ii]) x_start_by_order.append(x_start_column_sort[ii]) x_end_by_order.append(x_end_column_sort[ii]-1) + else: + #print(x_start_without_mother,x_end_without_mother,peaks_neg_tot,'dodo') - columns_covered_by_mothers = [] + + columns_covered_by_mothers=[] + for dj in range(len(x_start_without_mother)): - columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_start_without_mother[dj], - x_end_without_mother[dj])) - columns_covered_by_mothers = list(set(columns_covered_by_mothers)) - - all_columns=np.arange(len(peaks_neg_tot)-1) - columns_not_covered=list(set(all_columns) - set(columns_covered_by_mothers)) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * (len(columns_not_covered) + len(x_start_without_mother))) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) - ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) - x_starting = np.append(x_starting, x_start_without_mother) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) - x_ending = np.append(x_ending, x_end_without_mother) - - columns_covered_by_with_child_no_mothers = [] + columns_covered_by_mothers=columns_covered_by_mothers+list(np.array(range(x_start_without_mother[dj],x_end_without_mother[dj])) ) + columns_covered_by_mothers=list(set(columns_covered_by_mothers)) + + all_columns=np.array(range(len(peaks_neg_tot)-1)) + + columns_not_covered=list( set(all_columns)-set(columns_covered_by_mothers) ) + + + y_type_2=list(y_type_2) + x_starting=list(x_starting) + x_ending=list(x_ending) + + for lj in columns_not_covered: + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(lj) + x_ending.append(lj+1) + ##y_lines_by_order.append(int(splitter_y_new[i])) + ##x_start_by_order.append(0) + for lk in range(len(x_start_without_mother)): + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(x_start_without_mother[lk]) + x_ending.append(x_end_without_mother[lk]) + + + y_type_2=np.array(y_type_2) + x_starting=np.array(x_starting) + x_ending=np.array(x_ending) + + columns_covered_by_with_child_no_mothers=[] + for dj in range(len(x_end_with_child_without_mother)): - columns_covered_by_with_child_no_mothers = columns_covered_by_with_child_no_mothers + \ - list(range(x_start_with_child_without_mother[dj], - x_end_with_child_without_mother[dj])) - columns_covered_by_with_child_no_mothers = list(set(columns_covered_by_with_child_no_mothers)) - - all_columns = np.arange(len(peaks_neg_tot)-1) - columns_not_covered_child_no_mother = list(set(all_columns) - set(columns_covered_by_with_child_no_mothers)) + columns_covered_by_with_child_no_mothers=columns_covered_by_with_child_no_mothers+list(np.array(range(x_start_with_child_without_mother[dj],x_end_with_child_without_mother[dj])) ) + columns_covered_by_with_child_no_mothers=list(set(columns_covered_by_with_child_no_mothers)) + + all_columns=np.array(range(len(peaks_neg_tot)-1)) + + columns_not_covered_child_no_mother=list( set(all_columns)-set(columns_covered_by_with_child_no_mothers) ) #indexes_to_be_spanned=[] - for i_s in range(len(x_end_with_child_without_mother)): + for i_s in range( len(x_end_with_child_without_mother) ): columns_not_covered_child_no_mother.append(x_start_with_child_without_mother[i_s]) - columns_not_covered_child_no_mother = np.sort(columns_not_covered_child_no_mother) - ind_args = np.arange(len(y_type_2)) - x_end_with_child_without_mother = np.array(x_end_with_child_without_mother) - x_start_with_child_without_mother = np.array(x_start_with_child_without_mother) + + + + columns_not_covered_child_no_mother=np.sort(columns_not_covered_child_no_mother) + + + ind_args=np.array(range(len(y_type_2))) + + + for i_s_nc in columns_not_covered_child_no_mother: if i_s_nc in x_start_with_child_without_mother: - x_end_biggest_column = x_end_with_child_without_mother[x_start_with_child_without_mother==i_s_nc][0] - args_all_biggest_lines = ind_args[(x_starting==i_s_nc) & - (x_ending==x_end_biggest_column)] - y_column_nc = y_type_2[args_all_biggest_lines] - x_start_column_nc = x_starting[args_all_biggest_lines] - x_end_column_nc = x_ending[args_all_biggest_lines] - y_column_nc = np.sort(y_column_nc) + x_end_biggest_column=np.array(x_end_with_child_without_mother)[np.array(x_start_with_child_without_mother)==i_s_nc][0] + args_all_biggest_lines=ind_args[(x_starting==i_s_nc) & (x_ending==x_end_biggest_column)] + + args_all_biggest_lines=np.array(args_all_biggest_lines) + y_column_nc=y_type_2[args_all_biggest_lines] + x_start_column_nc=x_starting[args_all_biggest_lines] + x_end_column_nc=x_ending[args_all_biggest_lines] + + y_column_nc=np.sort(y_column_nc) + for i_c in range(len(y_column_nc)): if i_c==(len(y_column_nc)-1): - ind_all_lines_between_nm_wc=ind_args[(y_type_2>y_column_nc[i_c]) & - (y_type_2=i_s_nc) & - (x_ending<=x_end_biggest_column)] + ind_all_lines_betweeen_nm_wc=ind_args[(y_type_2>y_column_nc[i_c]) & (y_type_2=i_s_nc) & (x_ending<=x_end_biggest_column)] else: - ind_all_lines_between_nm_wc=ind_args[(y_type_2>y_column_nc[i_c]) & - (y_type_2=i_s_nc) & - (x_ending<=x_end_biggest_column)] - y_all_between_nm_wc = y_type_2[ind_all_lines_between_nm_wc] - x_starting_all_between_nm_wc = x_starting[ind_all_lines_between_nm_wc] - x_ending_all_between_nm_wc = x_ending[ind_all_lines_between_nm_wc] + ind_all_lines_betweeen_nm_wc=ind_args[(y_type_2>y_column_nc[i_c]) & (y_type_2=i_s_nc) & (x_ending<=x_end_biggest_column)] + + y_all_between_nm_wc=y_type_2[ind_all_lines_betweeen_nm_wc] + x_starting_all_between_nm_wc=x_starting[ind_all_lines_betweeen_nm_wc] + x_ending_all_between_nm_wc=x_ending[ind_all_lines_betweeen_nm_wc] + + x_diff_all_between_nm_wc=x_ending_all_between_nm_wc-x_starting_all_between_nm_wc + - x_diff_all_between_nm_wc = x_ending_all_between_nm_wc - x_starting_all_between_nm_wc if len(x_diff_all_between_nm_wc)>0: biggest=np.argmax(x_diff_all_between_nm_wc) - - columns_covered_by_mothers = [] + + + columns_covered_by_mothers=[] + for dj in range(len(x_starting_all_between_nm_wc)): - columns_covered_by_mothers = columns_covered_by_mothers + \ - list(range(x_starting_all_between_nm_wc[dj], - x_ending_all_between_nm_wc[dj])) - columns_covered_by_mothers = list(set(columns_covered_by_mothers)) - - all_columns=np.arange(i_s_nc, x_end_biggest_column) - columns_not_covered = list(set(all_columns) - set(columns_covered_by_mothers)) + columns_covered_by_mothers=columns_covered_by_mothers+list(np.array(range(x_starting_all_between_nm_wc[dj],x_ending_all_between_nm_wc[dj])) ) + columns_covered_by_mothers=list(set(columns_covered_by_mothers)) + + all_columns=np.array(range(i_s_nc,x_end_biggest_column)) + + columns_not_covered=list( set(all_columns)-set(columns_covered_by_mothers) ) + should_longest_line_be_extended=0 - if (len(x_diff_all_between_nm_wc) > 0 and - set(list(range(x_starting_all_between_nm_wc[biggest], - x_ending_all_between_nm_wc[biggest])) + - list(columns_not_covered)) != set(all_columns)): + if len(x_diff_all_between_nm_wc)>0 and set( list( np.array(range(x_starting_all_between_nm_wc[biggest],x_ending_all_between_nm_wc[biggest])) )+list(columns_not_covered) ) !=set(all_columns): should_longest_line_be_extended=1 - index_lines_so_close_to_top_separator = \ - np.arange(len(y_all_between_nm_wc))[(y_all_between_nm_wc>y_column_nc[i_c]) & - (y_all_between_nm_wc<=(y_column_nc[i_c]+500))] - if len(index_lines_so_close_to_top_separator) > 0: - indexes_remained_after_deleting_closed_lines= \ - np.array(list(set(list(range(len(y_all_between_nm_wc)))) - - set(list(index_lines_so_close_to_top_separator)))) - if len(indexes_remained_after_deleting_closed_lines) > 0: - y_all_between_nm_wc = y_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] - x_starting_all_between_nm_wc = x_starting_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] - x_ending_all_between_nm_wc = x_ending_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + + index_lines_so_close_to_top_separator=np.array(range(len(y_all_between_nm_wc)))[(y_all_between_nm_wc>y_column_nc[i_c]) & (y_all_between_nm_wc<=(y_column_nc[i_c]+500))] + + + if len(index_lines_so_close_to_top_separator)>0: + indexes_remained_after_deleting_closed_lines= np.array( list ( set( list( np.array(range(len(y_all_between_nm_wc))) ) ) -set(list( index_lines_so_close_to_top_separator) ) ) ) + + if len(indexes_remained_after_deleting_closed_lines)>0: + y_all_between_nm_wc=y_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + x_starting_all_between_nm_wc=x_starting_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + x_ending_all_between_nm_wc=x_ending_all_between_nm_wc[indexes_remained_after_deleting_closed_lines] + + + y_all_between_nm_wc=list(y_all_between_nm_wc) + x_starting_all_between_nm_wc=list(x_starting_all_between_nm_wc) + x_ending_all_between_nm_wc=list(x_ending_all_between_nm_wc) + + + y_all_between_nm_wc.append(y_column_nc[i_c] ) + x_starting_all_between_nm_wc.append(i_s_nc) + x_ending_all_between_nm_wc.append(x_end_biggest_column) + - y_all_between_nm_wc = np.append(y_all_between_nm_wc, y_column_nc[i_c]) - x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, i_s_nc) - x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, x_end_biggest_column) - - if len(x_diff_all_between_nm_wc) > 0: + + + y_all_between_nm_wc=list(y_all_between_nm_wc) + x_starting_all_between_nm_wc=list(x_starting_all_between_nm_wc) + x_ending_all_between_nm_wc=list(x_ending_all_between_nm_wc) + + if len(x_diff_all_between_nm_wc)>0: try: - y_all_between_nm_wc = np.append(y_all_between_nm_wc, y_column_nc[i_c]) - x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, x_starting_all_between_nm_wc[biggest]) - x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, x_ending_all_between_nm_wc[biggest]) + x_starting_all_between_nm_wc.append(x_starting_all_between_nm_wc[biggest]) + x_ending_all_between_nm_wc.append(x_ending_all_between_nm_wc[biggest]) + y_all_between_nm_wc.append(y_column_nc[i_c]) except: pass - y_all_between_nm_wc = np.append(y_all_between_nm_wc, [y_column_nc[i_c]] * len(columns_not_covered)) - x_starting_all_between_nm_wc = np.append(x_starting_all_between_nm_wc, columns_not_covered) - x_ending_all_between_nm_wc = np.append(x_ending_all_between_nm_wc, np.array(columns_not_covered) + 1) - - ind_args_between=np.arange(len(x_ending_all_between_nm_wc)) - for column in range(i_s_nc, x_end_biggest_column): + + + for c_n_c in columns_not_covered: + y_all_between_nm_wc.append(y_column_nc[i_c]) + x_starting_all_between_nm_wc.append(c_n_c) + x_ending_all_between_nm_wc.append(c_n_c+1) + + y_all_between_nm_wc=np.array(y_all_between_nm_wc) + x_starting_all_between_nm_wc=np.array(x_starting_all_between_nm_wc) + x_ending_all_between_nm_wc=np.array(x_ending_all_between_nm_wc) + + ind_args_between=np.array(range(len(x_ending_all_between_nm_wc))) + + for column in range(i_s_nc,x_end_biggest_column): ind_args_in_col=ind_args_between[x_starting_all_between_nm_wc==column] #print('babali2') #print(ind_args_in_col,'ind_args_in_col') @@ -2032,7 +2053,14 @@ def return_boxes_of_images_by_order_of_reading_new( y_lines_by_order.append(y_col_sort[ii]) x_start_by_order.append(x_start_column_sort[ii]) x_end_by_order.append(x_end_column_sort[ii]-1) + + + + + + else: + #print(column,'column') ind_args_in_col=ind_args[x_starting==i_s_nc] #print('babali2') @@ -2053,25 +2081,28 @@ def return_boxes_of_images_by_order_of_reading_new( x_start_by_order.append(x_start_column_sort[ii]) x_end_by_order.append(x_end_column_sort[ii]-1) + + for il in range(len(y_lines_by_order)): - y_copy = list(y_lines_by_order) - x_start_copy = list(x_start_by_order) - x_end_copy = list(x_end_by_order) - + + + y_copy=list( np.copy(y_lines_by_order) ) + x_start_copy=list( np.copy(x_start_by_order) ) + x_end_copy=list ( np.copy(x_end_by_order) ) + #print(y_copy,'y_copy') y_itself=y_copy.pop(il) x_start_itself=x_start_copy.pop(il) x_end_itself=x_end_copy.pop(il) - + #print(y_copy,'y_copy2') - for column in range(x_start_itself, x_end_itself+1): + + for column in range(x_start_itself,x_end_itself+1): #print(column,'cols') y_in_cols=[] for yic in range(len(y_copy)): #print('burda') - if (y_copy[yic]>y_itself and - column>=x_start_copy[yic] and - column<=x_end_copy[yic]): + if y_copy[yic]>y_itself and column>=x_start_copy[yic] and column<=x_end_copy[yic]: y_in_cols.append(y_copy[yic]) #print('burda2') #print(y_in_cols,'y_in_cols') @@ -2079,52 +2110,77 @@ def return_boxes_of_images_by_order_of_reading_new( y_down=np.min(y_in_cols) else: y_down=[int(splitter_y_new[i+1])][0] - #print(y_itself,'y_itself') - boxes.append([peaks_neg_tot[column], - peaks_neg_tot[column+1], - y_itself, - y_down]) + #print(y_itself,'y_itself') + boxes.append([peaks_neg_tot[column],peaks_neg_tot[column+1],y_itself,y_down]) except: - boxes.append([0, peaks_neg_tot[len(peaks_neg_tot)-1], - int(splitter_y_new[i]), int(splitter_y_new[i+1])]) + boxes.append([0,peaks_neg_tot[len(peaks_neg_tot)-1],int(splitter_y_new[i]),int(splitter_y_new[i+1])]) + + + else: y_lines_by_order=[] x_start_by_order=[] x_end_by_order=[] if len(x_starting)>0: - all_columns = np.arange(len(peaks_neg_tot)-1) - columns_covered_by_lines_covered_more_than_2col = [] + all_columns = np.array(range(len(peaks_neg_tot)-1)) + columns_covered_by_lines_covered_more_than_2col=[] + for dj in range(len(x_starting)): - if set(list(range(x_starting[dj],x_ending[dj]))) == set(all_columns): + if set( list(np.array(range(x_starting[dj],x_ending[dj])) ) ) == set(all_columns): pass else: - columns_covered_by_lines_covered_more_than_2col = columns_covered_by_lines_covered_more_than_2col + \ - list(range(x_starting[dj],x_ending[dj])) - columns_covered_by_lines_covered_more_than_2col = list(set(columns_covered_by_lines_covered_more_than_2col)) - columns_not_covered = list(set(all_columns) - set(columns_covered_by_lines_covered_more_than_2col)) - - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * (len(columns_not_covered) + 1)) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) - ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) - if len(new_main_sep_y) > 0: - x_starting = np.append(x_starting, 0) - x_ending = np.append(x_ending, len(peaks_neg_tot)-1) - else: - x_starting = np.append(x_starting, x_starting[0]) - x_ending = np.append(x_ending, x_ending[0]) + columns_covered_by_lines_covered_more_than_2col=columns_covered_by_lines_covered_more_than_2col+list(np.array(range(x_starting[dj],x_ending[dj])) ) + columns_covered_by_lines_covered_more_than_2col=list(set(columns_covered_by_lines_covered_more_than_2col)) + + + + columns_not_covered=list( set(all_columns)-set(columns_covered_by_lines_covered_more_than_2col) ) + + + y_type_2=list(y_type_2) + x_starting=list(x_starting) + x_ending=list(x_ending) + + for lj in columns_not_covered: + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(lj) + x_ending.append(lj+1) + ##y_lines_by_order.append(int(splitter_y_new[i])) + ##x_start_by_order.append(0) + + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(x_starting[0]) + x_ending.append(x_ending[0]) + + + y_type_2=np.array(y_type_2) + x_starting=np.array(x_starting) + x_ending=np.array(x_ending) else: - all_columns = np.arange(len(peaks_neg_tot)-1) - columns_not_covered = list(set(all_columns)) - y_type_2 = np.append(y_type_2, [int(splitter_y_new[i])] * len(columns_not_covered)) - ##y_lines_by_order = np.append(y_lines_by_order, [int(splitter_y_new[i])] * len(columns_not_covered)) - ##x_start_by_order = np.append(x_start_by_order, [0] * len(columns_not_covered)) - x_starting = np.append(x_starting, columns_not_covered) - x_ending = np.append(x_ending, np.array(columns_not_covered) + 1) - + all_columns=np.array(range(len(peaks_neg_tot)-1)) + columns_not_covered=list( set(all_columns) ) + + + y_type_2=list(y_type_2) + x_starting=list(x_starting) + x_ending=list(x_ending) + + for lj in columns_not_covered: + y_type_2.append(int(splitter_y_new[i])) + x_starting.append(lj) + x_ending.append(lj+1) + ##y_lines_by_order.append(int(splitter_y_new[i])) + ##x_start_by_order.append(0) + + + + y_type_2=np.array(y_type_2) + x_starting=np.array(x_starting) + x_ending=np.array(x_ending) + ind_args=np.array(range(len(y_type_2))) #ind_args=np.array(ind_args) + #print(ind_args,'ind_args') for column in range(len(peaks_neg_tot)-1): #print(column,'column') ind_args_in_col=ind_args[x_starting==column] @@ -2144,26 +2200,28 @@ def return_boxes_of_images_by_order_of_reading_new( y_lines_by_order.append(y_col_sort[ii]) x_start_by_order.append(x_start_column_sort[ii]) x_end_by_order.append(x_end_column_sort[ii]-1) - + + for il in range(len(y_lines_by_order)): - y_copy = list(y_lines_by_order) - x_start_copy = list(x_start_by_order) - x_end_copy = list(x_end_by_order) - + + + y_copy=list( np.copy(y_lines_by_order) ) + x_start_copy=list( np.copy(x_start_by_order) ) + x_end_copy=list ( np.copy(x_end_by_order) ) + #print(y_copy,'y_copy') y_itself=y_copy.pop(il) x_start_itself=x_start_copy.pop(il) x_end_itself=x_end_copy.pop(il) - + #print(y_copy,'y_copy2') - for column in range(x_start_itself, x_end_itself+1): + + for column in range(x_start_itself,x_end_itself+1): #print(column,'cols') y_in_cols=[] for yic in range(len(y_copy)): #print('burda') - if (y_copy[yic]>y_itself and - column>=x_start_copy[yic] and - column<=x_end_copy[yic]): + if y_copy[yic]>y_itself and column>=x_start_copy[yic] and column<=x_end_copy[yic]: y_in_cols.append(y_copy[yic]) #print('burda2') #print(y_in_cols,'y_in_cols') @@ -2171,27 +2229,12 @@ def return_boxes_of_images_by_order_of_reading_new( y_down=np.min(y_in_cols) else: y_down=[int(splitter_y_new[i+1])][0] - #print(y_itself,'y_itself') - boxes.append([peaks_neg_tot[column], - peaks_neg_tot[column+1], - y_itself, - y_down]) + #print(y_itself,'y_itself') + boxes.append([peaks_neg_tot[column],peaks_neg_tot[column+1],y_itself,y_down]) + + + #else: #boxes.append([ 0, regions_without_separators[:,:].shape[1] ,splitter_y_new[i],splitter_y_new[i+1]]) - if right2left_readingorder: - peaks_neg_tot_tables_new = [] - if len(peaks_neg_tot_tables)>=1: - for peaks_tab_ind in peaks_neg_tot_tables: - peaks_neg_tot_tables_ind = regions_without_separators.shape[1] - np.array(peaks_tab_ind) - peaks_neg_tot_tables_ind = list(peaks_neg_tot_tables_ind[::-1]) - peaks_neg_tot_tables_new.append(peaks_neg_tot_tables_ind) - - for i in range(len(boxes)): - x_start_new = regions_without_separators.shape[1] - boxes[i][1] - x_end_new = regions_without_separators.shape[1] - boxes[i][0] - boxes[i][0] = x_start_new - boxes[i][1] = x_end_new - return boxes, peaks_neg_tot_tables_new - else: - return boxes, peaks_neg_tot_tables + return boxes diff --git a/qurator/eynollah/utils/contour.py b/qurator/eynollah/utils/contour.py new file mode 100644 index 0000000..3209731 --- /dev/null +++ b/qurator/eynollah/utils/contour.py @@ -0,0 +1,238 @@ +import cv2 +import numpy as np +from shapely import geometry + +from .rotate import rotate_image, rotation_image_new + +def contours_in_same_horizon(cy_main_hor): + X1 = np.zeros((len(cy_main_hor), len(cy_main_hor))) + X2 = np.zeros((len(cy_main_hor), len(cy_main_hor))) + + X1[0::1, :] = cy_main_hor[:] + X2 = X1.T + + X_dif = np.abs(X2 - X1) + args_help = np.array(range(len(cy_main_hor))) + all_args = [] + for i in range(len(cy_main_hor)): + list_h = list(args_help[X_dif[i, :] <= 20]) + list_h.append(i) + if len(list_h) > 1: + all_args.append(list(set(list_h))) + return np.unique(all_args) + +def find_contours_mean_y_diff(contours_main): + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + return np.mean(np.diff(np.sort(np.array(cy_main)))) + + +def get_text_region_boxes_by_given_contours(contours): + + kernel = np.ones((5, 5), np.uint8) + boxes = [] + contours_new = [] + for jj in range(len(contours)): + x, y, w, h = cv2.boundingRect(contours[jj]) + + boxes.append([x, y, w, h]) + contours_new.append(contours[jj]) + + del contours + return boxes, contours_new + +def filter_contours_area_of_image(image, contours, hierarchy, max_area, min_area): + found_polygons_early = list() + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + area = polygon.area + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]) and hierarchy[0][jv][3] == -1: # and hierarchy[0][jv][3]==-1 : + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.uint)) + jv += 1 + return found_polygons_early + +def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, min_area): + found_polygons_early = list() + + jv = 0 + for c in contours: + if len(c) < 3: # A polygon cannot have less than 3 points + continue + + polygon = geometry.Polygon([point[0] for point in c]) + # area = cv2.contourArea(c) + area = polygon.area + ##print(np.prod(thresh.shape[:2])) + # Check that polygon has area greater than minimal area + # print(hierarchy[0][jv][3],hierarchy ) + if area >= min_area * np.prod(image.shape[:2]) and area <= max_area * np.prod(image.shape[:2]): # and hierarchy[0][jv][3]==-1 : + # print(c[0][0][1]) + found_polygons_early.append(np.array([[point] for point in polygon.exterior.coords], dtype=np.int32)) + jv += 1 + return found_polygons_early + +def find_new_features_of_contours(contours_main): + + areas_main = np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) + M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] + cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] + try: + x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + argmin_x_main = np.array([np.argmin(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 0] for j in range(len(contours_main))]) + y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 1] for j in range(len(contours_main))]) + + x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) for j in range(len(contours_main))]) + except: + x_min_main = np.array([np.min(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + argmin_x_main = np.array([np.argmin(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0] for j in range(len(contours_main))]) + y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 1] for j in range(len(contours_main))]) + + x_max_main = np.array([np.max(contours_main[j][:, 0]) for j in range(len(contours_main))]) + + y_min_main = np.array([np.min(contours_main[j][:, 1]) for j in range(len(contours_main))]) + y_max_main = np.array([np.max(contours_main[j][:, 1]) for j in range(len(contours_main))]) + + # dis_x=np.abs(x_max_main-x_min_main) + + return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin + +def return_parent_contours(contours, hierarchy): + contours_parent = [contours[i] for i in range(len(contours)) if hierarchy[0][i][3] == -1] + return contours_parent + +def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hierarchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_area) + + return contours_imgs + +def get_textregion_contours_in_org_image(cnts, img, slope_first): + + cnts_org = [] + # print(cnts,'cnts') + for i in range(len(cnts)): + img_copy = np.zeros(img.shape) + img_copy = cv2.fillPoly(img_copy, pts=[cnts[i]], color=(1, 1, 1)) + + # plt.imshow(img_copy) + # plt.show() + + # print(img.shape,'img') + img_copy = rotation_image_new(img_copy, -slope_first) + ##print(img_copy.shape,'img_copy') + # plt.imshow(img_copy) + # plt.show() + + img_copy = img_copy.astype(np.uint8) + imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) + cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) + # print(np.shape(cont_int[0])) + cnts_org.append(cont_int[0]) + + # print(cnts_org,'cnts_org') + + # sys.exit() + # self.y_shift = np.abs(img_copy.shape[0] - img.shape[0]) + # self.x_shift = np.abs(img_copy.shape[1] - img.shape[1]) + return cnts_org + +def return_contours_of_interested_textline(region_pre_p, pixel): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hierarchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=0.000000003) + return contours_imgs + +def return_contours_of_image(image): + + if len(image.shape) == 2: + image = np.repeat(image[:, :, np.newaxis], 3, axis=2) + image = image.astype(np.uint8) + else: + image = image.astype(np.uint8) + imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + return contours, hierarchy + +def return_contours_of_interested_region_by_min_size(region_pre_p, pixel, min_size=0.00003): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hierarchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=1, min_area=min_size) + + return contours_imgs + +def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, max_area): + + # pixels of images are identified by 5 + if len(region_pre_p.shape) == 3: + cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 + else: + cnts_images = (region_pre_p[:, :] == pixel) * 1 + cnts_images = cnts_images.astype(np.uint8) + cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) + imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + contours_imgs = return_parent_contours(contours_imgs, hierarchy) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=max_area, min_area=min_area) + + img_ret = np.zeros((region_pre_p.shape[0], region_pre_p.shape[1], 3)) + img_ret = cv2.fillPoly(img_ret, pts=contours_imgs, color=(1, 1, 1)) + return img_ret[:, :, 0] + diff --git a/src/eynollah/utils/counter.py b/qurator/eynollah/utils/counter.py similarity index 85% rename from src/eynollah/utils/counter.py rename to qurator/eynollah/utils/counter.py index 9a3ed70..bc1d765 100644 --- a/src/eynollah/utils/counter.py +++ b/qurator/eynollah/utils/counter.py @@ -7,13 +7,13 @@ class EynollahIdCounter(): def __init__(self, region_idx=0, line_idx=0): self._counter = Counter() - self._initial_region_idx = region_idx - self._initial_line_idx = line_idx + self._inital_region_idx = region_idx + self._inital_line_idx = line_idx self.reset() def reset(self): - self.set('region', self._initial_region_idx) - self.set('line', self._initial_line_idx) + self.set('region', self._inital_region_idx) + self.set('line', self._inital_line_idx) def inc(self, name, val=1): self._counter.update({name: val}) diff --git a/src/eynollah/utils/drop_capitals.py b/qurator/eynollah/utils/drop_capitals.py similarity index 64% rename from src/eynollah/utils/drop_capitals.py rename to qurator/eynollah/utils/drop_capitals.py index 67547d3..a69e9f5 100644 --- a/src/eynollah/utils/drop_capitals.py +++ b/qurator/eynollah/utils/drop_capitals.py @@ -4,7 +4,6 @@ from .contour import ( find_new_features_of_contours, return_contours_of_image, return_parent_contours, - return_contours_of_interested_region, ) def adhere_drop_capital_region_into_corresponding_textline( @@ -14,14 +13,13 @@ def adhere_drop_capital_region_into_corresponding_textline( contours_only_text_parent_h, all_box_coord, all_box_coord_h, - all_found_textline_polygons, - all_found_textline_polygons_h, + all_found_texline_polygons, + all_found_texline_polygons_h, kernel=None, curved_line=False, - textline_light=False, ): - # print(np.shape(all_found_textline_polygons),np.shape(all_found_textline_polygons[3]),'all_found_textline_polygonsshape') - # print(all_found_textline_polygons[3]) + # print(np.shape(all_found_texline_polygons),np.shape(all_found_texline_polygons[3]),'all_found_texline_polygonsshape') + # print(all_found_texline_polygons[3]) cx_m, cy_m, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent) cx_h, cy_h, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent_h) cx_d, cy_d, _, _, y_min_d, y_max_d, _ = find_new_features_of_contours(polygons_of_drop_capitals) @@ -78,7 +76,7 @@ def adhere_drop_capital_region_into_corresponding_textline( # region_with_intersected_drop=region_with_intersected_drop/3 region_with_intersected_drop = region_with_intersected_drop.astype(np.uint8) # print(np.unique(img_con_all_copy[:,:,0])) - if curved_line or textline_light: + if curved_line: if len(region_with_intersected_drop) > 1: sum_pixels_of_intersection = [] @@ -89,9 +87,9 @@ def adhere_drop_capital_region_into_corresponding_textline( region_final = region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)] - 1 # print(region_final,'region_final') - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) try: - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -107,26 +105,21 @@ def adhere_drop_capital_region_into_corresponding_textline( arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) # print(arg_min) - cnt_nearest = np.copy(all_found_textline_polygons[int(region_final)][arg_min]) - cnt_nearest[:, 0, 0] = all_found_textline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] - cnt_nearest[:, 0, 1] = all_found_textline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] + cnt_nearest[:, 0, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) img_textlines = img_textlines.astype(np.uint8) - - contours_combined = return_contours_of_interested_region(img_textlines, 255, 0) - - #plt.imshow(img_textlines) - #plt.show() - - #imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) - #ret, thresh = cv2.threshold(imgray, 0, 255, 0) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) - #contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + # print(len(contours_combined),'len textlines mixed') areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) contours_biggest = contours_combined[np.argmax(areas_cnt_text)] @@ -137,13 +130,8 @@ def adhere_drop_capital_region_into_corresponding_textline( # contours_biggest[:,0,1]=contours_biggest[:,0,1]#-all_box_coord[int(region_final)][0] # contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) - - if len(contours_combined)==1: - all_found_textline_polygons[int(region_final)][arg_min] = contours_biggest - elif len(contours_combined)==2: - all_found_textline_polygons[int(region_final)].insert(arg_min, polygons_of_drop_capitals[i_drop] ) - else: - pass + + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest except: # print('gordun1') @@ -151,64 +139,59 @@ def adhere_drop_capital_region_into_corresponding_textline( elif len(region_with_intersected_drop) == 1: region_final = region_with_intersected_drop[0] - 1 - # areas_main=np.array([cv2.contourArea(all_found_textline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_textline_polygons[int(region_final)]))]) + # areas_main=np.array([cv2.contourArea(all_found_texline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_texline_polygons[int(region_final)]))]) - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) + + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) + # print(all_box_coord[j_cont]) + # print(cx_t) + # print(cy_t) + # print(cx_d[i_drop]) + # print(cy_d[i_drop]) + y_lines = np.array(cy_t) # all_box_coord[int(region_final)][0]+np.array(cy_t) + + y_lines[y_lines < y_min_d[i_drop]] = 0 + # print(y_lines) + + arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) + # print(arg_min) + + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] + cnt_nearest[:, 0, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] + + img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) + img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) + img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) + + img_textlines = img_textlines.astype(np.uint8) + + # plt.imshow(img_textlines) + # plt.show() + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) + + contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + + # print(len(contours_combined),'len textlines mixed') + areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) + + contours_biggest = contours_combined[np.argmax(areas_cnt_text)] + + # print(np.shape(contours_biggest)) + # print(contours_biggest[:]) + # contours_biggest[:,0,0]=contours_biggest[:,0,0]#-all_box_coord[int(region_final)][2] + # contours_biggest[:,0,1]=contours_biggest[:,0,1]#-all_box_coord[int(region_final)][0] + # print(np.shape(contours_biggest),'contours_biggest') + # print(np.shape(all_found_texline_polygons[int(region_final)][arg_min])) + ##contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + + # print(cx_t,'print') try: - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) - # print(all_box_coord[j_cont]) - # print(cx_t) - # print(cy_t) - # print(cx_d[i_drop]) - # print(cy_d[i_drop]) - y_lines = np.array(cy_t) # all_box_coord[int(region_final)][0]+np.array(cy_t) - - y_lines[y_lines < y_min_d[i_drop]] = 0 - # print(y_lines) - - arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) - # print(arg_min) - - cnt_nearest = np.copy(all_found_textline_polygons[int(region_final)][arg_min]) - cnt_nearest[:, 0, 0] = all_found_textline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] - cnt_nearest[:, 0, 1] = all_found_textline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] - - img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) - img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) - img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) - - img_textlines = img_textlines.astype(np.uint8) - - - contours_combined = return_contours_of_interested_region(img_textlines, 255, 0) - ##imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) - ##ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - ##contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) - - contours_biggest = contours_combined[np.argmax(areas_cnt_text)] - - # print(np.shape(contours_biggest)) - # print(contours_biggest[:]) - # contours_biggest[:,0,0]=contours_biggest[:,0,0]#-all_box_coord[int(region_final)][2] - # contours_biggest[:,0,1]=contours_biggest[:,0,1]#-all_box_coord[int(region_final)][0] - # print(np.shape(contours_biggest),'contours_biggest') - # print(np.shape(all_found_textline_polygons[int(region_final)][arg_min])) - ##contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) - if len(contours_combined)==1: - all_found_textline_polygons[int(region_final)][arg_min] = contours_biggest - elif len(contours_combined)==2: - all_found_textline_polygons[int(region_final)].insert(arg_min, polygons_of_drop_capitals[i_drop] ) - else: - pass - except: - pass - - try: - # print(all_found_textline_polygons[j_cont][0]) - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # print(all_found_texline_polygons[j_cont][0]) + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -222,20 +205,19 @@ def adhere_drop_capital_region_into_corresponding_textline( arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) # print(arg_min) - cnt_nearest = np.copy(all_found_textline_polygons[int(region_final)][arg_min]) - cnt_nearest[:, 0, 0] = all_found_textline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] - cnt_nearest[:, 0, 1] = all_found_textline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 0] # +all_box_coord[int(region_final)][2] + cnt_nearest[:, 0, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 0, 1] # +all_box_coord[int(region_final)][0] img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) img_textlines = img_textlines.astype(np.uint8) - contours_combined = return_contours_of_interested_region(img_textlines, 255, 0) - #imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) - #ret, thresh = cv2.threshold(imgray, 0, 255, 0) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) - #contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # print(len(contours_combined),'len textlines mixed') areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) @@ -248,20 +230,15 @@ def adhere_drop_capital_region_into_corresponding_textline( contours_biggest[:, 0, 1] = contours_biggest[:, 0, 1] # -all_box_coord[int(region_final)][0] ##contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) - if len(contours_combined)==1: - all_found_textline_polygons[int(region_final)][arg_min] = contours_biggest - elif len(contours_combined)==2: - all_found_textline_polygons[int(region_final)].insert(arg_min, polygons_of_drop_capitals[i_drop] ) - else: - pass - # all_found_textline_polygons[int(region_final)][arg_min]=contours_biggest + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + # all_found_texline_polygons[int(region_final)][arg_min]=contours_biggest except: pass else: pass - ##cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + ##cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) ###print(all_box_coord[j_cont]) ###print(cx_t) ###print(cy_t) @@ -275,9 +252,9 @@ def adhere_drop_capital_region_into_corresponding_textline( ##arg_min=np.argmin(np.abs(y_lines-y_min_d[i_drop]) ) ###print(arg_min) - ##cnt_nearest=np.copy(all_found_textline_polygons[int(region_final)][arg_min]) - ##cnt_nearest[:,0,0]=all_found_textline_polygons[int(region_final)][arg_min][:,0,0]#+all_box_coord[int(region_final)][2] - ##cnt_nearest[:,0,1]=all_found_textline_polygons[int(region_final)][arg_min][:,0,1]#+all_box_coord[int(region_final)][0] + ##cnt_nearest=np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + ##cnt_nearest[:,0,0]=all_found_texline_polygons[int(region_final)][arg_min][:,0,0]#+all_box_coord[int(region_final)][2] + ##cnt_nearest[:,0,1]=all_found_texline_polygons[int(region_final)][arg_min][:,0,1]#+all_box_coord[int(region_final)][0] ##img_textlines=np.zeros((text_regions_p.shape[0],text_regions_p.shape[1],3)) ##img_textlines=cv2.fillPoly(img_textlines,pts=[cnt_nearest],color=(255,255,255)) @@ -303,7 +280,7 @@ def adhere_drop_capital_region_into_corresponding_textline( ##contours_biggest[:,0,1]=contours_biggest[:,0,1]#-all_box_coord[int(region_final)][0] ##contours_biggest=contours_biggest.reshape(np.shape(contours_biggest)[0],np.shape(contours_biggest)[2]) - ##all_found_textline_polygons[int(region_final)][arg_min]=contours_biggest + ##all_found_texline_polygons[int(region_final)][arg_min]=contours_biggest else: if len(region_with_intersected_drop) > 1: @@ -315,9 +292,9 @@ def adhere_drop_capital_region_into_corresponding_textline( region_final = region_with_intersected_drop[np.argmax(sum_pixels_of_intersection)] - 1 # print(region_final,'region_final') - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) try: - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -333,21 +310,19 @@ def adhere_drop_capital_region_into_corresponding_textline( arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) # print(arg_min) - cnt_nearest = np.copy(all_found_textline_polygons[int(region_final)][arg_min]) - cnt_nearest[:, 0] = all_found_textline_polygons[int(region_final)][arg_min][:, 0] + all_box_coord[int(region_final)][2] - cnt_nearest[:, 1] = all_found_textline_polygons[int(region_final)][arg_min][:, 1] + all_box_coord[int(region_final)][0] + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0] + all_box_coord[int(region_final)][2] + cnt_nearest[:, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 1] + all_box_coord[int(region_final)][0] img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) img_textlines = img_textlines.astype(np.uint8) - contours_combined = return_contours_of_interested_region(img_textlines, 255, 0) - - #imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) - #ret, thresh = cv2.threshold(imgray, 0, 255, 0) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) - #contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # print(len(contours_combined),'len textlines mixed') areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) @@ -360,12 +335,8 @@ def adhere_drop_capital_region_into_corresponding_textline( contours_biggest[:, 0, 1] = contours_biggest[:, 0, 1] - all_box_coord[int(region_final)][0] contours_biggest = contours_biggest.reshape(np.shape(contours_biggest)[0], np.shape(contours_biggest)[2]) - if len(contours_combined)==1: - all_found_textline_polygons[int(region_final)][arg_min] = contours_biggest - elif len(contours_combined)==2: - all_found_textline_polygons[int(region_final)].insert(arg_min, polygons_of_drop_capitals[i_drop] ) - else: - pass + + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest except: # print('gordun1') @@ -373,14 +344,14 @@ def adhere_drop_capital_region_into_corresponding_textline( elif len(region_with_intersected_drop) == 1: region_final = region_with_intersected_drop[0] - 1 - # areas_main=np.array([cv2.contourArea(all_found_textline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_textline_polygons[int(region_final)]))]) + # areas_main=np.array([cv2.contourArea(all_found_texline_polygons[int(region_final)][0][j] ) for j in range(len(all_found_texline_polygons[int(region_final)]))]) - # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) # print(cx_t,'print') try: - # print(all_found_textline_polygons[j_cont][0]) - cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_textline_polygons[int(region_final)]) + # print(all_found_texline_polygons[j_cont][0]) + cx_t, cy_t, _, _, _, _, _ = find_new_features_of_contours(all_found_texline_polygons[int(region_final)]) # print(all_box_coord[j_cont]) # print(cx_t) # print(cy_t) @@ -394,21 +365,19 @@ def adhere_drop_capital_region_into_corresponding_textline( arg_min = np.argmin(np.abs(y_lines - y_min_d[i_drop])) # print(arg_min) - cnt_nearest = np.copy(all_found_textline_polygons[int(region_final)][arg_min]) - cnt_nearest[:, 0] = all_found_textline_polygons[int(region_final)][arg_min][:, 0] + all_box_coord[int(region_final)][2] - cnt_nearest[:, 1] = all_found_textline_polygons[int(region_final)][arg_min][:, 1] + all_box_coord[int(region_final)][0] + cnt_nearest = np.copy(all_found_texline_polygons[int(region_final)][arg_min]) + cnt_nearest[:, 0] = all_found_texline_polygons[int(region_final)][arg_min][:, 0] + all_box_coord[int(region_final)][2] + cnt_nearest[:, 1] = all_found_texline_polygons[int(region_final)][arg_min][:, 1] + all_box_coord[int(region_final)][0] img_textlines = np.zeros((text_regions_p.shape[0], text_regions_p.shape[1], 3)) img_textlines = cv2.fillPoly(img_textlines, pts=[cnt_nearest], color=(255, 255, 255)) img_textlines = cv2.fillPoly(img_textlines, pts=[polygons_of_drop_capitals[i_drop]], color=(255, 255, 255)) img_textlines = img_textlines.astype(np.uint8) - contours_combined = return_contours_of_interested_region(img_textlines, 255, 0) - - #imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) - #ret, thresh = cv2.threshold(imgray, 0, 255, 0) + imgray = cv2.cvtColor(img_textlines, cv2.COLOR_BGR2GRAY) + ret, thresh = cv2.threshold(imgray, 0, 255, 0) - #contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) + contours_combined, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) # print(len(contours_combined),'len textlines mixed') areas_cnt_text = np.array([cv2.contourArea(contours_combined[j]) for j in range(len(contours_combined))]) @@ -421,13 +390,8 @@ def adhere_drop_capital_region_into_corresponding_textline( contours_biggest[:, 0, 1] = contours_biggest[:, 0, 1] - all_box_coord[int(region_final)][0] contours_biggest = contours_biggest.reshape(np.shape(contours_biggest)[0], np.shape(contours_biggest)[2]) - if len(contours_combined)==1: - all_found_textline_polygons[int(region_final)][arg_min] = contours_biggest - elif len(contours_combined)==2: - all_found_textline_polygons[int(region_final)].insert(arg_min, polygons_of_drop_capitals[i_drop] ) - else: - pass - # all_found_textline_polygons[int(region_final)][arg_min]=contours_biggest + all_found_texline_polygons[int(region_final)][arg_min] = contours_biggest + # all_found_texline_polygons[int(region_final)][arg_min]=contours_biggest except: pass @@ -452,8 +416,8 @@ def adhere_drop_capital_region_into_corresponding_textline( ######plt.show() #####try: #####if len(contours_new_parent)==1: - ######print(all_found_textline_polygons[j_cont][0]) - #####cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_textline_polygons[j_cont]) + ######print(all_found_texline_polygons[j_cont][0]) + #####cx_t,cy_t ,_, _, _ ,_,_= find_new_features_of_contours(all_found_texline_polygons[j_cont]) ######print(all_box_coord[j_cont]) ######print(cx_t) ######print(cy_t) @@ -466,9 +430,9 @@ def adhere_drop_capital_region_into_corresponding_textline( #####arg_min=np.argmin(np.abs(y_lines-y_min_d[i_drop]) ) ######print(arg_min) - #####cnt_nearest=np.copy(all_found_textline_polygons[j_cont][arg_min]) - #####cnt_nearest[:,0]=all_found_textline_polygons[j_cont][arg_min][:,0]+all_box_coord[j_cont][2] - #####cnt_nearest[:,1]=all_found_textline_polygons[j_cont][arg_min][:,1]+all_box_coord[j_cont][0] + #####cnt_nearest=np.copy(all_found_texline_polygons[j_cont][arg_min]) + #####cnt_nearest[:,0]=all_found_texline_polygons[j_cont][arg_min][:,0]+all_box_coord[j_cont][2] + #####cnt_nearest[:,1]=all_found_texline_polygons[j_cont][arg_min][:,1]+all_box_coord[j_cont][0] #####img_textlines=np.zeros((text_regions_p.shape[0],text_regions_p.shape[1],3)) #####img_textlines=cv2.fillPoly(img_textlines,pts=[cnt_nearest],color=(255,255,255)) @@ -489,7 +453,7 @@ def adhere_drop_capital_region_into_corresponding_textline( #####contours_biggest[:,0,0]=contours_biggest[:,0,0]-all_box_coord[j_cont][2] #####contours_biggest[:,0,1]=contours_biggest[:,0,1]-all_box_coord[j_cont][0] - #####all_found_textline_polygons[j_cont][arg_min]=contours_biggest + #####all_found_texline_polygons[j_cont][arg_min]=contours_biggest ######print(contours_biggest) ######plt.imshow(img_textlines[:,:,0]) ######plt.show() @@ -497,7 +461,7 @@ def adhere_drop_capital_region_into_corresponding_textline( #####pass #####except: #####pass - return all_found_textline_polygons + return all_found_texline_polygons def filter_small_drop_capitals_from_no_patch_layout(layout_no_patch, layout1): diff --git a/src/eynollah/utils/is_nan.py b/qurator/eynollah/utils/is_nan.py similarity index 100% rename from src/eynollah/utils/is_nan.py rename to qurator/eynollah/utils/is_nan.py diff --git a/src/eynollah/utils/marginals.py b/qurator/eynollah/utils/marginals.py similarity index 54% rename from src/eynollah/utils/marginals.py rename to qurator/eynollah/utils/marginals.py index a29e50d..7c43de6 100644 --- a/src/eynollah/utils/marginals.py +++ b/qurator/eynollah/utils/marginals.py @@ -2,11 +2,13 @@ import numpy as np import cv2 from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d + + from .contour import find_new_features_of_contours, return_contours_of_interested_region from .resize import resize_image from .rotate import rotate_image -def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_version=False, kernel=None): +def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, kernel=None): mask_marginals=np.zeros((text_with_lines.shape[0],text_with_lines.shape[1])) mask_marginals=mask_marginals.astype(np.uint8) @@ -47,14 +49,27 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve if thickness_along_y_percent>=14: text_with_lines_y_rev=-1*text_with_lines_y[:] + #print(text_with_lines_y) + #print(text_with_lines_y_rev) + + + + + #plt.plot(text_with_lines_y) + #plt.show() + text_with_lines_y_rev=text_with_lines_y_rev-np.min(text_with_lines_y_rev) + #plt.plot(text_with_lines_y_rev) + #plt.show() sigma_gaus=1 region_sum_0= gaussian_filter1d(text_with_lines_y, sigma_gaus) region_sum_0_rev=gaussian_filter1d(text_with_lines_y_rev, sigma_gaus) + #plt.plot(region_sum_0_rev) + #plt.show() region_sum_0_updown=region_sum_0[len(region_sum_0)::-1] first_nonzero=(next((i for i, x in enumerate(region_sum_0) if x), None)) @@ -63,18 +78,44 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve last_nonzero=len(region_sum_0)-last_nonzero + ##img_sum_0_smooth_rev=-region_sum_0 + + mid_point=(last_nonzero+first_nonzero)/2. one_third_right=(last_nonzero-mid_point)/3.0 one_third_left=(mid_point-first_nonzero)/3.0 - peaks, _ = find_peaks(text_with_lines_y_rev, height=0) - peaks=np.array(peaks) - peaks=peaks[(peaks>first_nonzero) & ((peaksfirst_nonzero) & ((peaksmid_point] peaks_left=peaks[peaks=mask_marginals.shape[1]: point_right=mask_marginals.shape[1]-1 @@ -104,8 +148,10 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve except: mask_marginals[:,:]=1 + #print(mask_marginals.shape,point_left,point_right,'nadosh') mask_marginals_rotated=rotate_image(mask_marginals,-slope_deskew) + #print(mask_marginals_rotated.shape,'nadosh') mask_marginals_rotated_sum=mask_marginals_rotated.sum(axis=0) mask_marginals_rotated_sum[mask_marginals_rotated_sum!=0]=1 @@ -121,92 +167,73 @@ def get_marginals(text_with_lines, text_regions, num_col, slope_deskew, light_ve if max_point_of_right_marginal>=text_regions.shape[1]: max_point_of_right_marginal=text_regions.shape[1]-1 - if light_version: - text_regions_org = np.copy(text_regions) - text_regions[text_regions[:,:]==1]=4 - - pixel_img=4 - min_area_text=0.00001 - - polygon_mask_marginals_rotated = return_contours_of_interested_region(mask_marginals,1,min_area_text) - - polygon_mask_marginals_rotated = polygon_mask_marginals_rotated[0] - polygons_of_marginals=return_contours_of_interested_region(text_regions,pixel_img,min_area_text) + #print(np.min(index_x_interest) ,np.max(index_x_interest),'minmaxnew') + #print(mask_marginals_rotated.shape,text_regions.shape,'mask_marginals_rotated') + #plt.imshow(mask_marginals) + #plt.show() - cx_text_only,cy_text_only ,x_min_text_only,x_max_text_only, y_min_text_only ,y_max_text_only,y_cor_x_min_main=find_new_features_of_contours(polygons_of_marginals) + #plt.imshow(mask_marginals_rotated) + #plt.show() - text_regions[(text_regions[:,:]==4)]=1 + text_regions[(mask_marginals_rotated[:,:]!=1) & (text_regions[:,:]==1)]=4 - marginlas_should_be_main_text=[] + #plt.imshow(text_regions) + #plt.show() - x_min_marginals_left=[] - x_min_marginals_right=[] + pixel_img=4 + min_area_text=0.00001 + polygons_of_marginals=return_contours_of_interested_region(text_regions,pixel_img,min_area_text) - for i in range(len(cx_text_only)): - results = cv2.pointPolygonTest(polygon_mask_marginals_rotated, (cx_text_only[i], cy_text_only[i]), False) + cx_text_only,cy_text_only ,x_min_text_only,x_max_text_only, y_min_text_only ,y_max_text_only,y_cor_x_min_main=find_new_features_of_contours(polygons_of_marginals) - if results == -1: - marginlas_should_be_main_text.append(polygons_of_marginals[i]) + text_regions[(text_regions[:,:]==4)]=1 + marginlas_should_be_main_text=[] + x_min_marginals_left=[] + x_min_marginals_right=[] - text_regions_org=cv2.fillPoly(text_regions_org, pts =marginlas_should_be_main_text, color=(4,4)) - text_regions = np.copy(text_regions_org) - + for i in range(len(cx_text_only)): - else: - - text_regions[(mask_marginals_rotated[:,:]!=1) & (text_regions[:,:]==1)]=4 - - pixel_img=4 - min_area_text=0.00001 - - polygons_of_marginals=return_contours_of_interested_region(text_regions,pixel_img,min_area_text) - - cx_text_only,cy_text_only ,x_min_text_only,x_max_text_only, y_min_text_only ,y_max_text_only,y_cor_x_min_main=find_new_features_of_contours(polygons_of_marginals) - - text_regions[(text_regions[:,:]==4)]=1 - - marginlas_should_be_main_text=[] - - x_min_marginals_left=[] - x_min_marginals_right=[] - - for i in range(len(cx_text_only)): - x_width_mar=abs(x_min_text_only[i]-x_max_text_only[i]) - y_height_mar=abs(y_min_text_only[i]-y_max_text_only[i]) - - if x_width_mar>16 and y_height_mar/x_width_mar<18: - marginlas_should_be_main_text.append(polygons_of_marginals[i]) - if x_min_text_only[i]<(mid_point-one_third_left): - x_min_marginals_left_new=x_min_text_only[i] - if len(x_min_marginals_left)==0: - x_min_marginals_left.append(x_min_marginals_left_new) - else: - x_min_marginals_left[0]=min(x_min_marginals_left[0],x_min_marginals_left_new) + x_width_mar=abs(x_min_text_only[i]-x_max_text_only[i]) + y_height_mar=abs(y_min_text_only[i]-y_max_text_only[i]) + #print(x_width_mar,y_height_mar,y_height_mar/x_width_mar,'y_height_mar') + if x_width_mar>16 and y_height_mar/x_width_mar<18: + marginlas_should_be_main_text.append(polygons_of_marginals[i]) + if x_min_text_only[i]<(mid_point-one_third_left): + x_min_marginals_left_new=x_min_text_only[i] + if len(x_min_marginals_left)==0: + x_min_marginals_left.append(x_min_marginals_left_new) else: - x_min_marginals_right_new=x_min_text_only[i] - if len(x_min_marginals_right)==0: - x_min_marginals_right.append(x_min_marginals_right_new) - else: - x_min_marginals_right[0]=min(x_min_marginals_right[0],x_min_marginals_right_new) + x_min_marginals_left[0]=min(x_min_marginals_left[0],x_min_marginals_left_new) + else: + x_min_marginals_right_new=x_min_text_only[i] + if len(x_min_marginals_right)==0: + x_min_marginals_right.append(x_min_marginals_right_new) + else: + x_min_marginals_right[0]=min(x_min_marginals_right[0],x_min_marginals_right_new) - if len(x_min_marginals_left)==0: - x_min_marginals_left=[0] - if len(x_min_marginals_right)==0: - x_min_marginals_right=[text_regions.shape[1]-1] + if len(x_min_marginals_left)==0: + x_min_marginals_left=[0] + if len(x_min_marginals_right)==0: + x_min_marginals_right=[text_regions.shape[1]-1] - text_regions=cv2.fillPoly(text_regions, pts =marginlas_should_be_main_text, color=(4,4)) - #text_regions[:,:int(x_min_marginals_left[0])][text_regions[:,:int(x_min_marginals_left[0])]==1]=0 - #text_regions[:,int(x_min_marginals_right[0]):][text_regions[:,int(x_min_marginals_right[0]):]==1]=0 - - - text_regions[:,:int(min_point_of_left_marginal)][text_regions[:,:int(min_point_of_left_marginal)]==1]=0 - text_regions[:,int(max_point_of_right_marginal):][text_regions[:,int(max_point_of_right_marginal):]==1]=0 + #print(x_min_marginals_left[0],x_min_marginals_right[0],'margo') + + #print(marginlas_should_be_main_text,'marginlas_should_be_main_text') + text_regions=cv2.fillPoly(text_regions, pts =marginlas_should_be_main_text, color=(4,4)) + + #print(np.unique(text_regions)) + + #text_regions[:,:int(x_min_marginals_left[0])][text_regions[:,:int(x_min_marginals_left[0])]==1]=0 + #text_regions[:,int(x_min_marginals_right[0]):][text_regions[:,int(x_min_marginals_right[0]):]==1]=0 + + text_regions[:,:int(min_point_of_left_marginal)][text_regions[:,:int(min_point_of_left_marginal)]==1]=0 + text_regions[:,int(max_point_of_right_marginal):][text_regions[:,int(max_point_of_right_marginal):]==1]=0 ###text_regions[:,0:point_left][text_regions[:,0:point_left]==1]=4 diff --git a/src/eynollah/utils/pil_cv2.py b/qurator/eynollah/utils/pil_cv2.py similarity index 80% rename from src/eynollah/utils/pil_cv2.py rename to qurator/eynollah/utils/pil_cv2.py index 9f6913e..20dc22f 100644 --- a/src/eynollah/utils/pil_cv2.py +++ b/qurator/eynollah/utils/pil_cv2.py @@ -1,4 +1,3 @@ -from contextlib import nullcontext from PIL import Image import numpy as np from ocrd_models import OcrdExif @@ -17,14 +16,13 @@ def pil2cv(img): def check_dpi(img): try: - if isinstance(img, Image.Image): - pil_image = nullcontext(img) + if isinstance(img, Image.__class__): + pil_image = img elif isinstance(img, str): pil_image = Image.open(img) else: - pil_image = nullcontext(cv2pil(img)) - with pil_image: - exif = OcrdExif(pil_image) + pil_image = cv2pil(img) + exif = OcrdExif(pil_image) resolution = exif.resolution if resolution == 1: raise Exception() diff --git a/src/eynollah/utils/resize.py b/qurator/eynollah/utils/resize.py similarity index 100% rename from src/eynollah/utils/resize.py rename to qurator/eynollah/utils/resize.py diff --git a/src/eynollah/utils/rotate.py b/qurator/eynollah/utils/rotate.py similarity index 79% rename from src/eynollah/utils/rotate.py rename to qurator/eynollah/utils/rotate.py index 189693d..9cadd4b 100644 --- a/src/eynollah/utils/rotate.py +++ b/qurator/eynollah/utils/rotate.py @@ -1,4 +1,6 @@ import math + +import imutils import cv2 def rotatedRectWithMaxArea(w, h, angle): @@ -33,14 +35,14 @@ def rotate_max_area_new(image, rotated, angle): return rotated[y1:y2, x1:x2] def rotation_image_new(img, thetha): - rotated = rotate_image(img, thetha) + rotated = imutils.rotate(img, thetha) return rotate_max_area_new(img, rotated, thetha) def rotate_image(img_patch, slope): (h, w) = img_patch.shape[:2] center = (w // 2, h // 2) M = cv2.getRotationMatrix2D(center, slope, 1.0) - return cv2.warpAffine(img_patch, M, (w, h) ) + return cv2.warpAffine(img_patch, M, (w, h), flags=cv2.INTER_CUBIC, borderMode=cv2.BORDER_REPLICATE) def rotate_image_different( img, slope): # img = cv2.imread('images/input.jpg') @@ -50,27 +52,26 @@ def rotate_image_different( img, slope): img_rotation = cv2.warpAffine(img, rotation_matrix, (num_cols, num_rows)) return img_rotation -def rotate_max_area(image, rotated, rotated_textline, rotated_layout, rotated_table_prediction, angle): +def rotate_max_area(image, rotated, rotated_textline, rotated_layout, angle): wr, hr = rotatedRectWithMaxArea(image.shape[1], image.shape[0], math.radians(angle)) h, w, _ = rotated.shape y1 = h // 2 - int(hr / 2) y2 = y1 + int(hr) x1 = w // 2 - int(wr / 2) x2 = x1 + int(wr) - return rotated[y1:y2, x1:x2], rotated_textline[y1:y2, x1:x2], rotated_layout[y1:y2, x1:x2], rotated_table_prediction[y1:y2, x1:x2] + return rotated[y1:y2, x1:x2], rotated_textline[y1:y2, x1:x2], rotated_layout[y1:y2, x1:x2] -def rotation_not_90_func(img, textline, text_regions_p_1, table_prediction, thetha): - rotated = rotate_image(img, thetha) - rotated_textline = rotate_image(textline, thetha) - rotated_layout = rotate_image(text_regions_p_1, thetha) - rotated_table_prediction = rotate_image(table_prediction, thetha) - return rotate_max_area(img, rotated, rotated_textline, rotated_layout, rotated_table_prediction, thetha) +def rotation_not_90_func(img, textline, text_regions_p_1, thetha): + rotated = imutils.rotate(img, thetha) + rotated_textline = imutils.rotate(textline, thetha) + rotated_layout = imutils.rotate(text_regions_p_1, thetha) + return rotate_max_area(img, rotated, rotated_textline, rotated_layout, thetha) def rotation_not_90_func_full_layout(img, textline, text_regions_p_1, text_regions_p_fully, thetha): - rotated = rotate_image(img, thetha) - rotated_textline = rotate_image(textline, thetha) - rotated_layout = rotate_image(text_regions_p_1, thetha) - rotated_layout_full = rotate_image(text_regions_p_fully, thetha) + rotated = imutils.rotate(img, thetha) + rotated_textline = imutils.rotate(textline, thetha) + rotated_layout = imutils.rotate(text_regions_p_1, thetha) + rotated_layout_full = imutils.rotate(text_regions_p_fully, thetha) return rotate_max_area_full_layout(img, rotated, rotated_textline, rotated_layout, rotated_layout_full, thetha) def rotate_max_area_full_layout(image, rotated, rotated_textline, rotated_layout, rotated_layout_full, angle): diff --git a/src/eynollah/utils/separate_lines.py b/qurator/eynollah/utils/separate_lines.py similarity index 71% rename from src/eynollah/utils/separate_lines.py rename to qurator/eynollah/utils/separate_lines.py index 0322579..acdc2e9 100644 --- a/src/eynollah/utils/separate_lines.py +++ b/qurator/eynollah/utils/separate_lines.py @@ -1,23 +1,20 @@ -import os -from logging import getLogger -from functools import partial import numpy as np import cv2 from scipy.signal import find_peaks from scipy.ndimage import gaussian_filter1d +import os + from .rotate import rotate_image -from .resize import resize_image from .contour import ( return_parent_contours, filter_contours_area_of_image_tables, return_contours_of_image, - filter_contours_area_of_image, - return_contours_of_interested_textline, - find_contours_mean_y_diff, + filter_contours_area_of_image ) +from .is_nan import isNaN from . import ( find_num_col_deskew, - crop_image_inside_box, + isNaN, ) def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): @@ -41,7 +38,9 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): y_max_cont = img_patch.shape[0] xv = np.linspace(x_min_cont, x_max_cont, 1000) + textline_patch_sum_along_width = img_patch.sum(axis=axis) + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] @@ -50,8 +49,11 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): x = np.array(range(len(y))) peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) + if 1 > 0: + try: + y_padded_smoothed_e = gaussian_filter1d(y_padded, 2) y_padded_up_to_down_e = -y_padded + np.max(y_padded) y_padded_up_to_down_padded_e = np.zeros(len(y_padded_up_to_down_e) + 40) @@ -62,7 +64,7 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) neg_peaks_max = np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] + arg_neg_must_be_deleted = np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -73,11 +75,12 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): clusters_to_be_deleted = [] if len(arg_diff_cluster) > 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : - arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] for m in range(len(clusters_to_be_deleted)): @@ -87,6 +90,7 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): for m1 in range(len(clusters_to_be_deleted[m])): peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1] - 1]] peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1]]] + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg_e[clusters_to_be_deleted[m][m1]]] peaks_new_tot = [] for i1 in peaks_new: @@ -99,18 +103,16 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): peaks_new_tot = peaks_e[:] textline_con, hierarchy = return_contours_of_image(img_patch) - textline_con_fil = filter_contours_area_of_image(img_patch, - textline_con, hierarchy, - max_area=1, min_area=0.0008) - if len(np.diff(peaks_new_tot))>1: - y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) - sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) - else: - sigma_gaus = 12 + textline_con_fil = filter_contours_area_of_image(img_patch, textline_con, hierarchy, max_area=1, min_area=0.0008) + y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + + sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) + # print(sigma_gaus,'sigma_gaus') except: sigma_gaus = 12 if sigma_gaus < 3: sigma_gaus = 3 + # print(sigma_gaus,'sigma') y_padded_smoothed = gaussian_filter1d(y_padded, sigma_gaus) y_padded_up_to_down = -y_padded + np.max(y_padded) @@ -121,23 +123,16 @@ def dedup_separate_lines(img_patch, contour_text_interest, thetha, axis): peaks, _ = find_peaks(y_padded_smoothed, height=0) peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) - return (x, y, - x_d, y_d, - xv, - x_min_cont, y_min_cont, - x_max_cont, y_max_cont, - first_nonzero, - y_padded_up_to_down_padded, - y_padded_smoothed, - peaks, peaks_neg, - rotation_matrix) + return x, y, x_d, y_d, xv, x_min_cont, y_min_cont, x_max_cont, y_max_cont, first_nonzero, y_padded_up_to_down_padded, y_padded_smoothed, peaks, peaks_neg, rotation_matrix def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): + (h, w) = img_patch.shape[:2] center = (w // 2, h // 2) M = cv2.getRotationMatrix2D(center, -thetha, 1.0) x_d = M[0, 2] y_d = M[1, 2] + thetha = thetha / 180. * np.pi rotation_matrix = np.array([[np.cos(thetha), -np.sin(thetha)], [np.sin(thetha), np.cos(thetha)]]) contour_text_interest_copy = contour_text_interest.copy() @@ -153,7 +148,9 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): y_max_cont = img_patch.shape[0] xv = np.linspace(x_min_cont, x_max_cont, 1000) + textline_patch_sum_along_width = img_patch.sum(axis=1) + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] @@ -163,101 +160,131 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) - try: - y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) - y_padded_up_to_down_e=-y_padded+np.max(y_padded) - y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) - y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e - y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) - - peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) - peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) - neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + if 1>0: - arg_neg_must_be_deleted= np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3] - diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) - - arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) - arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - peaks_new=peaks_e[:] - peaks_neg_new=peaks_neg_e[:] + try: - clusters_to_be_deleted=[] - if len(arg_diff_cluster)>0: - clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) - for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1: - arg_diff_cluster[i+1]+1]) - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) - if len(clusters_to_be_deleted)>0: - peaks_new_extra=[] - for m in range(len(clusters_to_be_deleted)): - min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) - max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) - peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) - for m1 in range(len(clusters_to_be_deleted[m])): - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] - peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] - peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] - peaks_new_tot=[] - for i1 in peaks_new: - peaks_new_tot.append(i1) - for i1 in peaks_new_extra: - peaks_new_tot.append(i1) - peaks_new_tot=np.sort(peaks_new_tot) - else: - peaks_new_tot=peaks_e[:] - - textline_con,hierarchy=return_contours_of_image(img_patch) - textline_con_fil=filter_contours_area_of_image(img_patch, - textline_con, hierarchy, - max_area=1, min_area=0.0008) - - if len(np.diff(peaks_new_tot))>0: - y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) - sigma_gaus=int( y_diff_mean * (7./40.0) ) - else: - sigma_gaus=12 + y_padded_smoothed_e= gaussian_filter1d(y_padded, 2) + y_padded_up_to_down_e=-y_padded+np.max(y_padded) + y_padded_up_to_down_padded_e=np.zeros(len(y_padded_up_to_down_e)+40) + y_padded_up_to_down_padded_e[20:len(y_padded_up_to_down_e)+20]=y_padded_up_to_down_e + y_padded_up_to_down_padded_e= gaussian_filter1d(y_padded_up_to_down_padded_e, 2) - except: - sigma_gaus=12 - if sigma_gaus<3: - sigma_gaus=3 + + peaks_e, _ = find_peaks(y_padded_smoothed_e, height=0) + peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) + neg_peaks_max=np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) + + arg_neg_must_be_deleted= np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e]/float(neg_peaks_max)<0.3 ] + diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) + + + + arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) + arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] + + + peaks_new=peaks_e[:] + peaks_neg_new=peaks_neg_e[:] + + clusters_to_be_deleted=[] + if len(arg_diff_cluster)>0: + + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) + for i in range(len(arg_diff_cluster)-1): + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) + + + if len(clusters_to_be_deleted)>0: + peaks_new_extra=[] + for m in range(len(clusters_to_be_deleted)): + min_cluster=np.min(peaks_e[clusters_to_be_deleted[m]]) + max_cluster=np.max(peaks_e[clusters_to_be_deleted[m]]) + peaks_new_extra.append( int( (min_cluster+max_cluster)/2.0) ) + for m1 in range(len(clusters_to_be_deleted[m])): + peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]-1]] + peaks_new=peaks_new[peaks_new!=peaks_e[clusters_to_be_deleted[m][m1]]] + + peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg_e[clusters_to_be_deleted[m][m1]]] + peaks_new_tot=[] + for i1 in peaks_new: + peaks_new_tot.append(i1) + for i1 in peaks_new_extra: + peaks_new_tot.append(i1) + peaks_new_tot=np.sort(peaks_new_tot) + + + else: + peaks_new_tot=peaks_e[:] + + + textline_con,hierarchy=return_contours_of_image(img_patch) + textline_con_fil=filter_contours_area_of_image(img_patch,textline_con,hierarchy,max_area=1,min_area=0.0008) + y_diff_mean=np.mean(np.diff(peaks_new_tot))#self.find_contours_mean_y_diff(textline_con_fil) + + sigma_gaus=int( y_diff_mean * (7./40.0) ) + #print(sigma_gaus,'sigma_gaus') + except: + sigma_gaus=12 + if sigma_gaus<3: + sigma_gaus=3 + #print(sigma_gaus,'sigma') + y_padded_smoothed= gaussian_filter1d(y_padded, sigma_gaus) y_padded_up_to_down=-y_padded+np.max(y_padded) y_padded_up_to_down_padded=np.zeros(len(y_padded_up_to_down)+40) y_padded_up_to_down_padded[20:len(y_padded_up_to_down)+20]=y_padded_up_to_down y_padded_up_to_down_padded= gaussian_filter1d(y_padded_up_to_down_padded, sigma_gaus) + + peaks, _ = find_peaks(y_padded_smoothed, height=0) peaks_neg, _ = find_peaks(y_padded_up_to_down_padded, height=0) + + + + try: neg_peaks_max=np.max(y_padded_smoothed[peaks]) - arg_neg_must_be_deleted= np.arange(len(peaks_neg))[y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.42] + + + arg_neg_must_be_deleted= np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg]/float(neg_peaks_max)<0.42 ] + + diff_arg_neg_must_be_deleted=np.diff(arg_neg_must_be_deleted) + + arg_diff=np.array(range(len(diff_arg_neg_must_be_deleted))) arg_diff_cluster=arg_diff[diff_arg_neg_must_be_deleted>1] - except: arg_neg_must_be_deleted=[] arg_diff_cluster=[] + + try: peaks_new=peaks[:] peaks_neg_new=peaks_neg[:] clusters_to_be_deleted=[] + + if len(arg_diff_cluster)>=2 and len(arg_diff_cluster)>0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0:arg_diff_cluster[0]+1]) for i in range(len(arg_diff_cluster)-1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1: - arg_diff_cluster[i+1]+1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i]+1:arg_diff_cluster[i+1]+1]) clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster)-1]+1:]) elif len(arg_neg_must_be_deleted)>=2 and len(arg_diff_cluster)==0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) + + if len(arg_neg_must_be_deleted)==1: clusters_to_be_deleted.append(arg_neg_must_be_deleted) + + if len(clusters_to_be_deleted)>0: peaks_new_extra=[] for m in range(len(clusters_to_be_deleted)): @@ -267,6 +294,7 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): for m1 in range(len(clusters_to_be_deleted[m])): peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]-1]] peaks_new=peaks_new[peaks_new!=peaks[clusters_to_be_deleted[m][m1]]] + peaks_neg_new=peaks_neg_new[peaks_neg_new!=peaks_neg[clusters_to_be_deleted[m][m1]]] peaks_new_tot=[] for i1 in peaks_new: @@ -275,29 +303,51 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): peaks_new_tot.append(i1) peaks_new_tot=np.sort(peaks_new_tot) + ##plt.plot(y_padded_up_to_down_padded) + ##plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') + ##plt.show() + + ##plt.plot(y_padded_up_to_down_padded) + ##plt.plot(peaks_neg_new,y_padded_up_to_down_padded[peaks_neg_new],'*') + ##plt.show() + + ##plt.plot(y_padded_smoothed) + ##plt.plot(peaks,y_padded_smoothed[peaks],'*') + ##plt.show() + + ##plt.plot(y_padded_smoothed) + ##plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') + ##plt.show() + peaks=peaks_new_tot[:] peaks_neg=peaks_neg_new[:] + + else: peaks_new_tot=peaks[:] peaks=peaks_new_tot[:] peaks_neg=peaks_neg_new[:] except: pass - if len(y_padded_smoothed[peaks]) > 1: - mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks=np.std(y_padded_smoothed[peaks]) - else: - mean_value_of_peaks = np.nan - std_value_of_peaks = np.nan + + + mean_value_of_peaks=np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks=np.std(y_padded_smoothed[peaks]) peaks_values=y_padded_smoothed[peaks] + + peaks_neg = peaks_neg - 20 - 20 peaks = peaks - 20 + for jj in range(len(peaks_neg)): if peaks_neg[jj] > len(x) - 1: peaks_neg[jj] = len(x) - 1 + for jj in range(len(peaks)): if peaks[jj] > len(x) - 1: peaks[jj] = len(x) - 1 + + textline_boxes = [] textline_boxes_rot = [] @@ -332,6 +382,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_down_narrow = peaks[jj] + first_nonzero + int( 1.1 * dis_to_next_down) ###-int(dis_to_next_down*1./2) + + if point_down_narrow >= img_patch.shape[0]: point_down_narrow = img_patch.shape[0] - 2 @@ -368,6 +420,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): if point_up_rot2<0: point_up_rot2=0 + + x_min_rot1=x_min_rot1-x_help x_max_rot2=x_max_rot2-x_help x_max_rot3=x_max_rot3-x_help @@ -378,24 +432,29 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_down_rot3=point_down_rot3-y_help point_down_rot4=point_down_rot4-y_help + + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + elif len(peaks) < 1: pass elif len(peaks) == 1: - distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[0] + first_nonzero])), True) - for mj in range(len(xv))] + distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[0] + first_nonzero])), True) + for mj in range(len(xv))] distances = np.array(distances) xvinside = xv[distances >= 0] + if len(xvinside) == 0: x_min = x_min_cont x_max = x_max_cont @@ -418,6 +477,7 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + if x_min_rot1<0: x_min_rot1=0 if x_min_rot4<0: @@ -426,6 +486,7 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_up_rot1=0 if point_up_rot2<0: point_up_rot2=0 + x_min_rot1=x_min_rot1-x_help x_max_rot2=x_max_rot2-x_help @@ -436,15 +497,22 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_up_rot2=point_up_rot2-y_help point_down_rot3=point_down_rot3-y_help point_down_rot4=point_down_rot4-y_help + + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + textline_boxes.append(np.array([[int(x_min), int(y_min)], [int(x_max), int(y_min)], [int(x_max), int(y_max)], [int(x_min), int(y_max)]])) + + + elif len(peaks) == 2: dis_to_next = np.abs(peaks[1] - peaks[0]) for jj in range(len(peaks)): @@ -462,12 +530,12 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): except: point_up =peaks[jj] + first_nonzero - int(1. / 1.8 * dis_to_next) - distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) - for mj in range(len(xv))] + distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + for mj in range(len(xv))] distances = np.array(distances) xvinside = xv[distances >= 0] + if len(xvinside) == 0: x_min = x_min_cont x_max = x_max_cont @@ -485,6 +553,8 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): x_max_rot3, point_down_rot3 = p3[0] + x_d, p3[1] + y_d x_min_rot4, point_down_rot4 = p4[0] + x_d, p4[1] + y_d + + if x_min_rot1<0: x_min_rot1=0 if x_min_rot4<0: @@ -504,16 +574,21 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_down_rot3=point_down_rot3-y_help point_down_rot4=point_down_rot4-y_help + + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) else: for jj in range(len(peaks)): + if jj == 0: dis_to_next = peaks[jj + 1] - peaks[jj] # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) @@ -537,12 +612,12 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_up = peaks[jj] + first_nonzero - int(1. / 1.9 * dis_to_next_up) point_down = peaks[jj] + first_nonzero + int(1. / 1.9 * dis_to_next_down) - distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) - for mj in range(len(xv))] + distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) + for mj in range(len(xv))] distances = np.array(distances) xvinside = xv[distances >= 0] + if len(xvinside) == 0: x_min = x_min_cont x_max = x_max_cont @@ -568,6 +643,7 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_up_rot1=0 if point_up_rot2<0: point_up_rot2=0 + x_min_rot1=x_min_rot1-x_help x_max_rot2=x_max_rot2-x_help @@ -578,23 +654,29 @@ def separate_lines(img_patch, contour_text_interest, thetha, x_help, y_help): point_up_rot2=point_up_rot2-y_help point_down_rot3=point_down_rot3-y_help point_down_rot4=point_down_rot4-y_help + + + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + + return peaks, textline_boxes_rot def separate_lines_vertical(img_patch, contour_text_interest, thetha): + thetha = thetha + 90 contour_text_interest_copy = contour_text_interest.copy() - x, y, x_d, y_d, xv, x_min_cont, y_min_cont, x_max_cont, y_max_cont, \ - first_nonzero, y_padded_up_to_down_padded, y_padded_smoothed, \ - peaks, peaks_neg, rotation_matrix = dedup_separate_lines(img_patch, contour_text_interest, thetha, 0) + x, y, x_d, y_d, xv, x_min_cont, y_min_cont, x_max_cont, y_max_cont, first_nonzero, y_padded_up_to_down_padded, y_padded_smoothed, peaks, peaks_neg, rotation_matrix = dedup_separate_lines(img_patch, contour_text_interest, thetha, 0) + # plt.plot(y_padded_up_to_down_padded) # plt.plot(peaks_neg,y_padded_up_to_down_padded[peaks_neg],'*') @@ -608,7 +690,8 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): neg_peaks_max = np.max(y_padded_up_to_down_padded[peaks_neg]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.42] + arg_neg_must_be_deleted = np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.42] + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -618,16 +701,18 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): peaks_neg_new = peaks_neg[:] clusters_to_be_deleted = [] - if len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) >= 2: + if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : - arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - else: + + if len(arg_neg_must_be_deleted) == 1: clusters_to_be_deleted.append(arg_neg_must_be_deleted) + if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] for m in range(len(clusters_to_be_deleted)): @@ -637,6 +722,7 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): for m1 in range(len(clusters_to_be_deleted[m])): peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1] - 1]] peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1]]] + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg[clusters_to_be_deleted[m][m1]]] peaks_new_tot = [] for i1 in peaks_new: @@ -652,14 +738,9 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): peaks_new_tot = peaks[:] peaks = peaks_new_tot[:] peaks_neg = peaks_neg_new[:] - - if len(y_padded_smoothed[peaks])>1: - mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks = np.std(y_padded_smoothed[peaks]) - else: - mean_value_of_peaks = np.nan - std_value_of_peaks = np.nan - + + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) peaks_values = y_padded_smoothed[peaks] peaks_neg = peaks_neg - 20 - 20 @@ -677,6 +758,7 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): textline_boxes_rot = [] if len(peaks_neg) == len(peaks) + 1 and len(peaks) >= 3: + # print('11') for jj in range(len(peaks)): if jj == (len(peaks) - 1): @@ -711,6 +793,7 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): distances = np.array(distances) xvinside = xv[distances >= 0] + if len(xvinside) == 0: x_min = x_min_cont x_max = x_max_cont @@ -737,16 +820,13 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): if point_up_rot2 < 0: point_up_rot2 = 0 - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + elif len(peaks) < 1: pass + elif len(peaks) == 1: x_min = x_min_cont x_max = x_max_cont @@ -773,14 +853,10 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): if point_up_rot2 < 0: point_up_rot2 = 0 - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - textline_boxes.append(np.array([[int(x_min), int(y_min)], - [int(x_max), int(y_min)], - [int(x_max), int(y_max)], - [int(x_min), int(y_max)]])) + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(y_min)], [int(x_max), int(y_min)], [int(x_max), int(y_max)], [int(x_min), int(y_max)]])) + elif len(peaks) == 2: dis_to_next = np.abs(peaks[1] - peaks[0]) for jj in range(len(peaks)): @@ -795,12 +871,11 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): point_down = img_patch.shape[0] - 2 point_up = peaks[jj] + first_nonzero - int(1.0 / 1.8 * dis_to_next) - distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) - for mj in range(len(xv))] + distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) for mj in range(len(xv))] distances = np.array(distances) xvinside = xv[distances >= 0] + if len(xvinside) == 0: x_min = x_min_cont x_max = x_max_cont @@ -827,16 +902,12 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): if point_up_rot2 < 0: point_up_rot2 = 0 - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) else: for jj in range(len(peaks)): + if jj == 0: dis_to_next = peaks[jj + 1] - peaks[jj] # point_up=peaks[jj]+first_nonzero-int(1./3*dis_to_next) @@ -860,12 +931,11 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): point_up = peaks[jj] + first_nonzero - int(1.0 / 1.9 * dis_to_next_up) point_down = peaks[jj] + first_nonzero + int(1.0 / 1.9 * dis_to_next_down) - distances = [cv2.pointPolygonTest(contour_text_interest_copy, - tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) - for mj in range(len(xv))] + distances = [cv2.pointPolygonTest(contour_text_interest_copy, tuple(int(x) for x in np.array([xv[mj], peaks[jj] + first_nonzero])), True) for mj in range(len(xv))] distances = np.array(distances) xvinside = xv[distances >= 0] + if len(xvinside) == 0: x_min = x_min_cont x_max = x_max_cont @@ -892,17 +962,14 @@ def separate_lines_vertical(img_patch, contour_text_interest, thetha): if point_up_rot2 < 0: point_up_rot2 = 0 - textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], - [int(x_max_rot2), int(point_up_rot2)], - [int(x_max_rot3), int(point_down_rot3)], - [int(x_min_rot4), int(point_down_rot4)]])) - textline_boxes.append(np.array([[int(x_min), int(point_up)], - [int(x_max), int(point_up)], - [int(x_max), int(point_down)], - [int(x_min), int(point_down)]])) + textline_boxes_rot.append(np.array([[int(x_min_rot1), int(point_up_rot1)], [int(x_max_rot2), int(point_up_rot2)], [int(x_max_rot3), int(point_down_rot3)], [int(x_min_rot4), int(point_down_rot4)]])) + + textline_boxes.append(np.array([[int(x_min), int(point_up)], [int(x_max), int(point_up)], [int(x_max), int(point_down)], [int(x_min), int(point_down)]])) + return peaks, textline_boxes_rot def separate_lines_new_inside_tiles2(img_patch, thetha): + (h, w) = img_patch.shape[:2] center = (w // 2, h // 2) M = cv2.getRotationMatrix2D(center, -thetha, 1.0) @@ -924,7 +991,9 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): y_max_cont = img_patch.shape[0] xv = np.linspace(x_min_cont, x_max_cont, 1000) + textline_patch_sum_along_width = img_patch.sum(axis=1) + first_nonzero = 0 # (next((i for i, x in enumerate(mada_n) if x), None)) y = textline_patch_sum_along_width[:] # [first_nonzero:last_nonzero] @@ -934,7 +1003,9 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): peaks_real, _ = find_peaks(gaussian_filter1d(y, 3), height=0) if 1 > 0: + try: + y_padded_smoothed_e = gaussian_filter1d(y_padded, 2) y_padded_up_to_down_e = -y_padded + np.max(y_padded) y_padded_up_to_down_padded_e = np.zeros(len(y_padded_up_to_down_e) + 40) @@ -945,7 +1016,7 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): peaks_neg_e, _ = find_peaks(y_padded_up_to_down_padded_e, height=0) neg_peaks_max = np.max(y_padded_up_to_down_padded_e[peaks_neg_e]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg_e))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] + arg_neg_must_be_deleted = np.array(range(len(peaks_neg_e)))[y_padded_up_to_down_padded_e[peaks_neg_e] / float(neg_peaks_max) < 0.3] diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) @@ -956,10 +1027,12 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): clusters_to_be_deleted = [] if len(arg_diff_cluster) > 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) + if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] for m in range(len(clusters_to_be_deleted)): @@ -969,6 +1042,7 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): for m1 in range(len(clusters_to_be_deleted[m])): peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1] - 1]] peaks_new = peaks_new[peaks_new != peaks_e[clusters_to_be_deleted[m][m1]]] + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg_e[clusters_to_be_deleted[m][m1]]] peaks_new_tot = [] for i1 in peaks_new: @@ -976,23 +1050,21 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): for i1 in peaks_new_extra: peaks_new_tot.append(i1) peaks_new_tot = np.sort(peaks_new_tot) + else: peaks_new_tot = peaks_e[:] textline_con, hierarchy = return_contours_of_image(img_patch) - textline_con_fil = filter_contours_area_of_image(img_patch, - textline_con, hierarchy, - max_area=1, min_area=0.0008) - if len(np.diff(peaks_new_tot)): - y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) - sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) - else: - sigma_gaus = 12 + textline_con_fil = filter_contours_area_of_image(img_patch, textline_con, hierarchy, max_area=1, min_area=0.0008) + y_diff_mean = np.mean(np.diff(peaks_new_tot)) # self.find_contours_mean_y_diff(textline_con_fil) + sigma_gaus = int(y_diff_mean * (7.0 / 40.0)) + # print(sigma_gaus,'sigma_gaus') except: sigma_gaus = 12 if sigma_gaus < 3: sigma_gaus = 3 + # print(sigma_gaus,'sigma') y_padded_smoothed = gaussian_filter1d(y_padded, sigma_gaus) y_padded_up_to_down = -y_padded + np.max(y_padded) @@ -1009,23 +1081,27 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): try: neg_peaks_max = np.max(y_padded_smoothed[peaks]) - arg_neg_must_be_deleted = np.arange(len(peaks_neg))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.24] + arg_neg_must_be_deleted = np.array(range(len(peaks_neg)))[y_padded_up_to_down_padded[peaks_neg] / float(neg_peaks_max) < 0.24] + diff_arg_neg_must_be_deleted = np.diff(arg_neg_must_be_deleted) arg_diff = np.array(range(len(diff_arg_neg_must_be_deleted))) arg_diff_cluster = arg_diff[diff_arg_neg_must_be_deleted > 1] clusters_to_be_deleted = [] - if len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) >= 2: + + if len(arg_diff_cluster) >= 2 and len(arg_diff_cluster) > 0: + clusters_to_be_deleted.append(arg_neg_must_be_deleted[0 : arg_diff_cluster[0] + 1]) for i in range(len(arg_diff_cluster) - 1): - clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : - arg_diff_cluster[i + 1] + 1]) + clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[i] + 1 : arg_diff_cluster[i + 1] + 1]) clusters_to_be_deleted.append(arg_neg_must_be_deleted[arg_diff_cluster[len(arg_diff_cluster) - 1] + 1 :]) elif len(arg_neg_must_be_deleted) >= 2 and len(arg_diff_cluster) == 0: clusters_to_be_deleted.append(arg_neg_must_be_deleted[:]) - else: + + if len(arg_neg_must_be_deleted) == 1: clusters_to_be_deleted.append(arg_neg_must_be_deleted) + if len(clusters_to_be_deleted) > 0: peaks_new_extra = [] for m in range(len(clusters_to_be_deleted)): @@ -1035,6 +1111,7 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): for m1 in range(len(clusters_to_be_deleted[m])): peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1] - 1]] peaks_new = peaks_new[peaks_new != peaks[clusters_to_be_deleted[m][m1]]] + peaks_neg_new = peaks_neg_new[peaks_neg_new != peaks_neg[clusters_to_be_deleted[m][m1]]] peaks_new_tot = [] for i1 in peaks_new: @@ -1058,6 +1135,7 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): # plt.plot(y_padded_smoothed) # plt.plot(peaks_new_tot,y_padded_smoothed[peaks_new_tot],'*') # plt.show() + peaks = peaks_new_tot[:] peaks_neg = peaks_neg_new[:] except: @@ -1067,14 +1145,9 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): peaks_new_tot = peaks[:] peaks = peaks_new_tot[:] peaks_neg = peaks_neg_new[:] - - if len(y_padded_smoothed[peaks]) > 1: - mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) - std_value_of_peaks = np.std(y_padded_smoothed[peaks]) - else: - mean_value_of_peaks = np.nan - std_value_of_peaks = np.nan - + + mean_value_of_peaks = np.mean(y_padded_smoothed[peaks]) + std_value_of_peaks = np.std(y_padded_smoothed[peaks]) peaks_values = y_padded_smoothed[peaks] ###peaks_neg = peaks_neg - 20 - 20 @@ -1084,10 +1157,13 @@ def separate_lines_new_inside_tiles2(img_patch, thetha): if len(peaks_neg_true) > 0: peaks_neg_true = np.array(peaks_neg_true) + peaks_neg_true = peaks_neg_true - 20 - 20 + # print(peaks_neg_true) for i in range(len(peaks_neg_true)): img_patch[peaks_neg_true[i] - 6 : peaks_neg_true[i] + 6, :] = 0 + else: pass @@ -1170,12 +1246,14 @@ def separate_lines_new_inside_tiles(img_path, thetha): if diff_peaks[i] <= cut_off: forest.append(peaks_neg[i + 1]) if diff_peaks[i] > cut_off: - if not np.isnan(forest[np.argmin(z[forest])]): + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) forest = [] forest.append(peaks_neg[i + 1]) if i == (len(peaks_neg) - 1): - if not np.isnan(forest[np.argmin(z[forest])]): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmin(z[forest])]): peaks_neg_true.append(forest[np.argmin(z[forest])]) diff_peaks_pos = np.abs(np.diff(peaks)) @@ -1191,14 +1269,17 @@ def separate_lines_new_inside_tiles(img_path, thetha): if diff_peaks_pos[i] <= cut_off: forest.append(peaks[i + 1]) if diff_peaks_pos[i] > cut_off: - if not np.isnan(forest[np.argmax(z[forest])]): + # print(forest[np.argmin(z[forest]) ] ) + if not isNaN(forest[np.argmax(z[forest])]): peaks_pos_true.append(forest[np.argmax(z[forest])]) forest = [] forest.append(peaks[i + 1]) if i == (len(peaks) - 1): - if not np.isnan(forest[np.argmax(z[forest])]): + # print(print(forest[np.argmin(z[forest]) ] )) + if not isNaN(forest[np.argmax(z[forest])]): peaks_pos_true.append(forest[np.argmax(z[forest])]) + # print(len(peaks_neg_true) ,len(peaks_pos_true) ,'lensss') if len(peaks_neg_true) > 0: peaks_neg_true = np.array(peaks_neg_true) @@ -1224,6 +1305,7 @@ def separate_lines_new_inside_tiles(img_path, thetha): """ peaks_neg_true = peaks_neg_true - 20 - 20 + # print(peaks_neg_true) for i in range(len(peaks_neg_true)): img_path[peaks_neg_true[i] - 6 : peaks_neg_true[i] + 6, :] = 0 @@ -1261,13 +1343,14 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables(thresh, - contours_imgs, hierarchy, - max_area=max_area, min_area=min_area) + contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, max_area=max_area, min_area=min_area) + cont_final = [] + ###print(add_boxes_coor_into_textlines,'ikki') for i in range(len(contours_imgs)): img_contour = np.zeros((cnts_images.shape[0], cnts_images.shape[1], 3)) img_contour = cv2.fillPoly(img_contour, pts=[contours_imgs[i]], color=(255, 255, 255)) + img_contour = img_contour.astype(np.uint8) img_contour = cv2.dilate(img_contour, kernel, iterations=4) @@ -1279,13 +1362,17 @@ def separate_lines_vertical_cont(img_patch, contour_text_interest, thetha, box_i ##0] ##contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] ##if add_boxes_coor_into_textlines: + ##print(np.shape(contours_text_rot[0]),'sjppo') ##contours_text_rot[0][:, 0, 0]=contours_text_rot[0][:, 0, 0] + box_ind[0] ##contours_text_rot[0][:, 0, 1]=contours_text_rot[0][:, 0, 1] + box_ind[1] cont_final.append(contours_text_rot[0]) + ##print(cont_final,'nadizzzz') return None, cont_final + def textline_contours_postprocessing(textline_mask, slope, contour_text_interest, box_ind, add_boxes_coor_into_textlines=False): + textline_mask = np.repeat(textline_mask[:, :, np.newaxis], 3, axis=2) * 255 textline_mask = textline_mask.astype(np.uint8) kernel = np.ones((5, 5), np.uint8) @@ -1293,14 +1380,25 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest textline_mask = cv2.morphologyEx(textline_mask, cv2.MORPH_CLOSE, kernel) textline_mask = cv2.erode(textline_mask, kernel, iterations=2) # textline_mask = cv2.erode(textline_mask, kernel, iterations=1) + + # print(textline_mask.shape[0]/float(textline_mask.shape[1]),'miz') try: + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(textline_mask) + # plt.show() + + # if abs(slope)>1: + # x_help=30 + # y_help=2 + # else: + # x_help=2 + # y_help=2 + x_help = 30 y_help = 2 - textline_mask_help = np.zeros((textline_mask.shape[0] + int(2 * y_help), - textline_mask.shape[1] + int(2 * x_help), 3)) - textline_mask_help[y_help : y_help + textline_mask.shape[0], - x_help : x_help + textline_mask.shape[1], :] = np.copy(textline_mask[:, :, :]) + textline_mask_help = np.zeros((textline_mask.shape[0] + int(2 * y_help), textline_mask.shape[1] + int(2 * x_help), 3)) + textline_mask_help[y_help : y_help + textline_mask.shape[0], x_help : x_help + textline_mask.shape[1], :] = np.copy(textline_mask[:, :, :]) dst = rotate_image(textline_mask_help, slope) dst = dst[:, :, 0] @@ -1311,19 +1409,36 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest # plt.show() contour_text_copy = contour_text_interest.copy() + contour_text_copy[:, 0, 0] = contour_text_copy[:, 0, 0] - box_ind[0] contour_text_copy[:, 0, 1] = contour_text_copy[:, 0, 1] - box_ind[1] img_contour = np.zeros((box_ind[3], box_ind[2], 3)) img_contour = cv2.fillPoly(img_contour, pts=[contour_text_copy], color=(255, 255, 255)) - img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), - img_contour.shape[1] + int(2 * x_help), 3)) - img_contour_help[y_help : y_help + img_contour.shape[0], - x_help : x_help + img_contour.shape[1], :] = np.copy(img_contour[:, :, :]) + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(img_contour) + # plt.show() + + img_contour_help = np.zeros((img_contour.shape[0] + int(2 * y_help), img_contour.shape[1] + int(2 * x_help), 3)) + + img_contour_help[y_help : y_help + img_contour.shape[0], x_help : x_help + img_contour.shape[1], :] = np.copy(img_contour[:, :, :]) img_contour_rot = rotate_image(img_contour_help, slope) + # plt.imshow(img_contour_rot_help) + # plt.show() + + # plt.imshow(dst_help) + # plt.show() + + # if np.abs(slope)>.5 and textline_mask.shape[0]/float(textline_mask.shape[1])>3: + # plt.imshow(img_contour_rot_help) + # plt.show() + + # plt.imshow(dst_help) + # plt.show() + img_contour_rot = img_contour_rot.astype(np.uint8) # dst_help = dst_help.astype(np.uint8) imgrayrot = cv2.cvtColor(img_contour_rot, cv2.COLOR_BGR2GRAY) @@ -1333,21 +1448,20 @@ def textline_contours_postprocessing(textline_mask, slope, contour_text_interest len_con_text_rot = [len(contours_text_rot[ib]) for ib in range(len(contours_text_rot))] ind_big_con = np.argmax(len_con_text_rot) + # print('juzaa') if abs(slope) > 45: - _, contours_rotated_clean = separate_lines_vertical_cont( - textline_mask, contours_text_rot[ind_big_con], box_ind, slope, - add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) + # print(add_boxes_coor_into_textlines,'avval') + _, contours_rotated_clean = separate_lines_vertical_cont(textline_mask, contours_text_rot[ind_big_con], box_ind, slope, add_boxes_coor_into_textlines=add_boxes_coor_into_textlines) else: - _, contours_rotated_clean = separate_lines( - dst, contours_text_rot[ind_big_con], slope, x_help, y_help) + _, contours_rotated_clean = separate_lines(dst, contours_text_rot[ind_big_con], slope, x_help, y_help) + except: + contours_rotated_clean = [] return contours_rotated_clean -def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, plotter=None): - if logger is None: - logger = getLogger(__package__) +def separate_lines_new2(img_path, thetha, num_col, slope_region, plotter=None): if num_col == 1: num_patches = int(img_path.shape[1] / 200.0) @@ -1365,11 +1479,14 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl length_x = int(img_path.shape[1] / float(num_patches)) # margin = int(0.04 * length_x) just recently this was changed because it break lines into 2 margin = int(0.04 * length_x) + # print(margin,'margin') # if margin<=4: # margin = int(0.08 * length_x) + # margin=0 width_mid = length_x - 2 * margin + nxf = img_path.shape[1] / float(width_mid) if nxf > int(nxf): @@ -1393,9 +1510,9 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl # img_patch = img[index_y_d:index_y_u, index_x_d:index_x_u, :] img_xline = img_patch_ineterst[:, index_x_d:index_x_u] + sigma = 2 try: - assert img_xline.any() - slope_xline = return_deskew_slop(img_xline, 2, logger=logger, plotter=plotter) + slope_xline = return_deskew_slop(img_xline, sigma, plotter=plotter) except: slope_xline = 0 @@ -1404,9 +1521,11 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl # if abs(slope_region)>70 and abs(slope_xline)<25: # slope_xline=[slope_region][0] slopes_tile_wise.append(slope_xline) + # print(slope_xline,'xlineeee') img_line_rotated = rotate_image(img_xline, slope_xline) img_line_rotated[:, :][img_line_rotated[:, :] != 0] = 1 - + + # print(slopes_tile_wise,'slopes_tile_wise') img_patch_ineterst = img_path[:, :] # [peaks_neg_true[14]-dis_up:peaks_neg_true[14]+dis_down ,:] img_patch_ineterst_revised = np.zeros(img_patch_ineterst.shape) @@ -1429,8 +1548,8 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl img_int[:, :] = img_xline[:, :] # img_patch_org[:,:,0] img_resized = np.zeros((int(img_int.shape[0] * (1.2)), int(img_int.shape[1] * (3)))) - img_resized[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], - int(img_int.shape[1] * (1.0)) : int(img_int.shape[1] * (1.0)) + img_int.shape[1]] = img_int[:, :] + + img_resized[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], int(img_int.shape[1] * (1)) : int(img_int.shape[1] * (1)) + img_int.shape[1]] = img_int[:, :] # plt.imshow(img_xline) # plt.show() img_line_rotated = rotate_image(img_resized, slopes_tile_wise[i]) @@ -1441,267 +1560,240 @@ def separate_lines_new2(img_path, thetha, num_col, slope_region, logger=None, pl img_patch_separated_returned = rotate_image(img_patch_separated, -slopes_tile_wise[i]) img_patch_separated_returned[:, :][img_patch_separated_returned[:, :] != 0] = 1 - img_patch_separated_returned_true_size = img_patch_separated_returned[ - int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], - int(img_int.shape[1] * (1.0)) : int(img_int.shape[1] * (1.0)) + img_int.shape[1]] + img_patch_separated_returned_true_size = img_patch_separated_returned[int(img_int.shape[0] * (0.1)) : int(img_int.shape[0] * (0.1)) + img_int.shape[0], int(img_int.shape[1] * (1)) : int(img_int.shape[1] * (1)) + img_int.shape[1]] img_patch_separated_returned_true_size = img_patch_separated_returned_true_size[:, margin : length_x - margin] img_patch_ineterst_revised[:, index_x_d + margin : index_x_u - margin] = img_patch_separated_returned_true_size + # plt.imshow(img_patch_ineterst_revised) + # plt.show() return img_patch_ineterst_revised -def do_image_rotation(angle, img, sigma_des, logger=None): - if logger is None: - logger = getLogger(__package__) - img_rot = rotate_image(img, angle) - img_rot[img_rot!=0] = 1 - try: - var = find_num_col_deskew(img_rot, sigma_des, 20.3) - except: - logger.exception("cannot determine variance for angle %.2f°", angle) - var = 0 - return var +def return_deskew_slop(img_patch_org, sigma_des, main_page=False, plotter=None): -def return_deskew_slop(img_patch_org, sigma_des,n_tot_angles=100, - main_page=False, logger=None, plotter=None, map=map): if main_page and plotter: plotter.save_plot_of_textline_density(img_patch_org) img_int=np.zeros((img_patch_org.shape[0],img_patch_org.shape[1])) img_int[:,:]=img_patch_org[:,:]#img_patch_org[:,:,0] + + max_shape=np.max(img_int.shape) img_resized=np.zeros((int( max_shape*(1.1) ) , int( max_shape*(1.1) ) )) + onset_x=int((img_resized.shape[1]-img_int.shape[1])/2.) onset_y=int((img_resized.shape[0]-img_int.shape[0])/2.) + #img_resized=np.zeros((int( img_int.shape[0]*(1.8) ) , int( img_int.shape[1]*(2.6) ) )) + + + #img_resized[ int( img_int.shape[0]*(.4)):int( img_int.shape[0]*(.4))+img_int.shape[0] , int( img_int.shape[1]*(.8)):int( img_int.shape[1]*(.8))+img_int.shape[1] ]=img_int[:,:] img_resized[ onset_y:onset_y+img_int.shape[0] , onset_x:onset_x+img_int.shape[1] ]=img_int[:,:] - if main_page and img_patch_org.shape[1] > img_patch_org.shape[0]: - angles = np.array([-45, 0, 45, 90,]) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + #print(img_resized.shape,'img_resizedshape') + #plt.imshow(img_resized) + #plt.show() - angles = np.linspace(angle - 22.5, angle + 22.5, n_tot_angles) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) - elif main_page: - angles = np.linspace(-12, 12, n_tot_angles)#np.array([0 , 45 , 90 , -45]) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + if main_page and img_patch_org.shape[1]>img_patch_org.shape[0]: + + #plt.imshow(img_resized) + #plt.show() + angels=np.array([-45, 0 , 45 , 90 , ])#np.linspace(-12,12,100)#np.array([0 , 45 , 90 , -45]) + + var_res=[] + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + #plt.imshow(img_rot) + #plt.show() + img_rot[img_rot!=0]=1 + #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(var_spectrum,'var_spectrum') + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + ##print(rot,var_spectrum,'var_spectrum') + except: + var_spectrum=0 + var_res.append(var_spectrum) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + + angels=np.linspace(ang_int-22.5,ang_int+22.5,100) + + var_res=[] + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + var_res.append(var_spectrum) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + elif main_page and img_patch_org.shape[1]<=img_patch_org.shape[0]: + + #plt.imshow(img_resized) + #plt.show() + angels=np.linspace(-12,12,100)#np.array([0 , 45 , 90 , -45]) + + + var_res=[] + + for rot in angels: + img_rot=rotate_image(img_resized,rot) + #plt.imshow(img_rot) + #plt.show() + img_rot[img_rot!=0]=1 + #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(var_spectrum,'var_spectrum') + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + + except: + var_spectrum=0 + + var_res.append(var_spectrum) + + + if plotter: + plotter.save_plot_of_rotation_angle(angels, var_res) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 early_slope_edge=11 - if abs(angle) > early_slope_edge: - if angle < 0: - angles = np.linspace(-90, -12, n_tot_angles) - else: - angles = np.linspace(90, 12, n_tot_angles) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + if abs(ang_int)>early_slope_edge and ang_int<0: + angels=np.linspace(-90,-12,100) + var_res=[] + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + var_res.append(var_spectrum) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + elif abs(ang_int)>early_slope_edge and ang_int>0: + + angels=np.linspace(90,12,100) + var_res=[] + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(indexer,'indexer') + except: + var_spectrum=0 + var_res.append(var_spectrum) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 else: - angles = np.linspace(-25, 25, int(0.5 * n_tot_angles) + 10) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + angels=np.linspace(-25,25,60) + var_res=[] + indexer=0 + for rot in angels: + img_rot=rotate_image(img_resized,rot) + #plt.imshow(img_rot) + #plt.show() + img_rot[img_rot!=0]=1 + #neg_peaks,var_spectrum=self.find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(var_spectrum,'var_spectrum') + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + var_res.append(var_spectrum) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + #plt.plot(var_res) + #plt.show() + ##plt.plot(mom3_res) + ##plt.show() + #print(ang_int,'ang_int111') early_slope_edge=22 - if abs(angle) > early_slope_edge: - if angle < 0: - angles = np.linspace(-90, -25, int(0.5 * n_tot_angles) + 10) - else: - angles = np.linspace(90, 25, int(0.5 * n_tot_angles) + 10) - angle = get_smallest_skew(img_resized, sigma_des, angles, map=map, logger=logger, plotter=plotter) + if abs(ang_int)>early_slope_edge and ang_int<0: - return angle + angels=np.linspace(-90,-25,60) -def get_smallest_skew(img, sigma_des, angles, logger=None, plotter=None, map=map): - if logger is None: - logger = getLogger(__package__) - results = list(map(partial(do_image_rotation, img=img, sigma_des=sigma_des, logger=logger), angles)) - if plotter: - plotter.save_plot_of_rotation_angle(angles, results) - try: - var_res = np.array(results) - assert var_res.any() - angle = angles[np.argmax(var_res)] - except: - logger.exception("cannot determine best angle among %s", str(angles)) - angle = 0 - return angle + var_res=[] -def do_work_of_slopes_new( - box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, image_page_rotated, slope_deskew, - logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None -): - if KERNEL is None: - KERNEL = np.ones((5, 5), np.uint8) - if logger is None: - logger = getLogger(__package__) - logger.debug('enter do_work_of_slopes_new') + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + except: + var_spectrum=0 + var_res.append(var_spectrum) - x, y, w, h = box_text - _, crop_coor = crop_image_inside_box(box_text, image_page_rotated) - mask_textline = np.zeros(textline_mask_tot_ea.shape) - mask_textline = cv2.fillPoly(mask_textline, pts=[contour], color=(1,1,1)) - all_text_region_raw = textline_mask_tot_ea * mask_textline - all_text_region_raw = all_text_region_raw[y: y + h, x: x + w].astype(np.uint8) - img_int_p = all_text_region_raw[:,:] - img_int_p = cv2.erode(img_int_p, KERNEL, iterations=2) - - if img_int_p.shape[0] /img_int_p.shape[1] < 0.1: - slope = 0 - slope_for_all = slope_deskew - all_text_region_raw = textline_mask_tot_ea[y: y + h, x: x + w] - cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text, 0) - else: - try: - textline_con, hierarchy = return_contours_of_image(img_int_p) - textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, - hierarchy, - max_area=1, min_area=0.00008) - y_diff_mean = find_contours_mean_y_diff(textline_con_fil) if len(textline_con_fil) > 1 else np.NaN - if np.isnan(y_diff_mean): - slope_for_all = MAX_SLOPE - else: - sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) - img_int_p[img_int_p > 0] = 1 - slope_for_all = return_deskew_slop(img_int_p, sigma_des, logger=logger, plotter=plotter) - if abs(slope_for_all) <= 0.5: - slope_for_all = slope_deskew - except: - logger.exception("cannot determine angle of contours") - slope_for_all = MAX_SLOPE - - if slope_for_all == MAX_SLOPE: - slope_for_all = slope_deskew - slope = slope_for_all - mask_only_con_region = np.zeros(textline_mask_tot_ea.shape) - mask_only_con_region = cv2.fillPoly(mask_only_con_region, pts=[contour_par], color=(1, 1, 1)) - - all_text_region_raw = textline_mask_tot_ea[y: y + h, x: x + w].copy() - mask_only_con_region = mask_only_con_region[y: y + h, x: x + w] - - all_text_region_raw[mask_only_con_region == 0] = 0 - cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text) - - return cnt_clean_rot, box_text, contour, contour_par, crop_coor, index_r_con, slope - -def do_work_of_slopes_new_curved( - box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, image_page_rotated, mask_texts_only, num_col, scale_par, slope_deskew, - logger=None, MAX_SLOPE=999, KERNEL=None, plotter=None -): - if KERNEL is None: - KERNEL = np.ones((5, 5), np.uint8) - if logger is None: - logger = getLogger(__package__) - logger.debug("enter do_work_of_slopes_new_curved") - - x, y, w, h = box_text - all_text_region_raw = textline_mask_tot_ea[y: y + h, x: x + w].astype(np.uint8) - img_int_p = all_text_region_raw[:, :] - - # img_int_p=cv2.erode(img_int_p,KERNEL,iterations = 2) - # plt.imshow(img_int_p) - # plt.show() - - if img_int_p.shape[0] / img_int_p.shape[1] < 0.1: - slope = 0 - slope_for_all = slope_deskew - else: - try: - textline_con, hierarchy = return_contours_of_image(img_int_p) - textline_con_fil = filter_contours_area_of_image(img_int_p, textline_con, - hierarchy, - max_area=1, min_area=0.0008) - y_diff_mean = find_contours_mean_y_diff(textline_con_fil) if len(textline_con_fil) > 1 else np.NaN - if np.isnan(y_diff_mean): - slope_for_all = MAX_SLOPE - else: - sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) - img_int_p[img_int_p > 0] = 1 - slope_for_all = return_deskew_slop(img_int_p, sigma_des, logger=logger, plotter=plotter) - if abs(slope_for_all) < 0.5: - slope_for_all = slope_deskew - except: - logger.exception("cannot determine angle of contours") - slope_for_all = MAX_SLOPE - - if slope_for_all == MAX_SLOPE: - slope_for_all = slope_deskew - slope = slope_for_all - - _, crop_coor = crop_image_inside_box(box_text, image_page_rotated) - - if abs(slope_for_all) < 45: - textline_region_in_image = np.zeros(textline_mask_tot_ea.shape) - x, y, w, h = cv2.boundingRect(contour_par) - mask_biggest = np.zeros(mask_texts_only.shape) - mask_biggest = cv2.fillPoly(mask_biggest, pts=[contour_par], color=(1, 1, 1)) - mask_region_in_patch_region = mask_biggest[y : y + h, x : x + w] - textline_biggest_region = mask_biggest * textline_mask_tot_ea - - textline_rotated_separated = separate_lines_new2(textline_biggest_region[y: y+h, x: x+w], 0, - num_col, slope_for_all, - logger=logger, plotter=plotter) - - - textline_rotated_separated[mask_region_in_patch_region[:, :] != 1] = 0 - - textline_region_in_image[y : y + h, x : x + w] = textline_rotated_separated - - - pixel_img = 1 - cnt_textlines_in_image = return_contours_of_interested_textline(textline_region_in_image, pixel_img) - - textlines_cnt_per_region = [] - for jjjj in range(len(cnt_textlines_in_image)): - mask_biggest2 = np.zeros(mask_texts_only.shape) - mask_biggest2 = cv2.fillPoly(mask_biggest2, pts=[cnt_textlines_in_image[jjjj]], color=(1, 1, 1)) - if num_col + 1 == 1: - mask_biggest2 = cv2.dilate(mask_biggest2, KERNEL, iterations=5) - else: - mask_biggest2 = cv2.dilate(mask_biggest2, KERNEL, iterations=4) - - pixel_img = 1 - mask_biggest2 = resize_image(mask_biggest2, int(mask_biggest2.shape[0] * scale_par), int(mask_biggest2.shape[1] * scale_par)) - cnt_textlines_in_image_ind = return_contours_of_interested_textline(mask_biggest2, pixel_img) try: - textlines_cnt_per_region.append(cnt_textlines_in_image_ind[0]) - except Exception as why: - logger.error(why) - else: - textlines_cnt_per_region = textline_contours_postprocessing(all_text_region_raw, slope_for_all, contour_par, box_text, True) + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 - return textlines_cnt_per_region[::-1], box_text, contour, contour_par, crop_coor, index_r_con, slope + elif abs(ang_int)>early_slope_edge and ang_int>0: -def do_work_of_slopes_new_light( - box_text, contour, contour_par, index_r_con, - textline_mask_tot_ea, image_page_rotated, slope_deskew, textline_light, - logger=None -): - if logger is None: - logger = getLogger(__package__) - logger.debug('enter do_work_of_slopes_new_light') + angels=np.linspace(90,25,60) - x, y, w, h = box_text - _, crop_coor = crop_image_inside_box(box_text, image_page_rotated) - mask_textline = np.zeros(textline_mask_tot_ea.shape) - mask_textline = cv2.fillPoly(mask_textline, pts=[contour], color=(1,1,1)) - all_text_region_raw = textline_mask_tot_ea * mask_textline - all_text_region_raw = all_text_region_raw[y: y + h, x: x + w].astype(np.uint8) + var_res=[] - mask_only_con_region = np.zeros(textline_mask_tot_ea.shape) - mask_only_con_region = cv2.fillPoly(mask_only_con_region, pts=[contour_par], color=(1, 1, 1)) + indexer=0 + for rot in angels: + img_rot=rotate_image(img_resized,rot) + ##plt.imshow(img_rot) + ##plt.show() + img_rot[img_rot!=0]=1 + try: + var_spectrum=find_num_col_deskew(img_rot,sigma_des,20.3 ) + #print(indexer,'indexer') + except: + var_spectrum=0 - if textline_light: - all_text_region_raw = np.copy(textline_mask_tot_ea) - all_text_region_raw[mask_only_con_region == 0] = 0 - cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(all_text_region_raw) - cnt_clean_rot = filter_contours_area_of_image(all_text_region_raw, cnt_clean_rot_raw, hir_on_cnt_clean_rot, - max_area=1, min_area=0.00001) - else: - all_text_region_raw = np.copy(textline_mask_tot_ea[y: y + h, x: x + w]) - mask_only_con_region = mask_only_con_region[y: y + h, x: x + w] - all_text_region_raw[mask_only_con_region == 0] = 0 - cnt_clean_rot = textline_contours_postprocessing(all_text_region_raw, slope_deskew, contour_par, box_text) + var_res.append(var_spectrum) + try: + var_res=np.array(var_res) + ang_int=angels[np.argmax(var_res)]#angels_sorted[arg_final]#angels[arg_sort_early[arg_sort[arg_final]]]#angels[arg_fin] + except: + ang_int=0 + + return ang_int - return cnt_clean_rot, box_text, contour, contour_par, crop_coor, index_r_con, slope_deskew diff --git a/src/eynollah/utils/xml.py b/qurator/eynollah/utils/xml.py similarity index 99% rename from src/eynollah/utils/xml.py rename to qurator/eynollah/utils/xml.py index bd95702..ac02190 100644 --- a/src/eynollah/utils/xml.py +++ b/qurator/eynollah/utils/xml.py @@ -21,6 +21,7 @@ from ocrd_models.ocrd_page import ( RegionRefType, SeparatorRegionType, TableRegionType, + TextEquivType, TextLineType, TextRegionType, UnorderedGroupIndexedType, @@ -72,7 +73,7 @@ def order_and_id_of_texts(found_polygons_text_region, found_polygons_text_region index_of_types_2 = index_of_types[kind_of_texts == 2] indexes_sorted_2 = indexes_sorted[kind_of_texts == 2] - + counter = EynollahIdCounter(region_idx=ref_point) for idx_textregion, _ in enumerate(found_polygons_text_region): id_of_texts.append(counter.next_region_id) diff --git a/src/eynollah/writer.py b/qurator/eynollah/writer.py similarity index 56% rename from src/eynollah/writer.py rename to qurator/eynollah/writer.py index 92e353f..8dfd2b2 100644 --- a/src/eynollah/writer.py +++ b/qurator/eynollah/writer.py @@ -2,7 +2,7 @@ # pylint: disable=import-error from pathlib import Path import os.path -import xml.etree.ElementTree as ET + from .utils.xml import create_page_xml, xml_reading_order from .utils.counter import EynollahIdCounter @@ -10,9 +10,9 @@ from ocrd_utils import getLogger from ocrd_models.ocrd_page import ( BorderType, CoordsType, + TextEquivType, PcGtsType, TextLineType, - TextEquivType, TextRegionType, ImageRegionType, TableRegionType, @@ -23,14 +23,12 @@ import numpy as np class EynollahXmlWriter(): - def __init__(self, *, dir_out, image_filename, curved_line,textline_light, pcgts=None): + def __init__(self, *, dir_out, image_filename, curved_line, pcgts=None): self.logger = getLogger('eynollah.writer') self.counter = EynollahIdCounter() self.dir_out = dir_out self.image_filename = image_filename - self.output_filename = os.path.join(self.dir_out or "", self.image_filename_stem) + ".xml" self.curved_line = curved_line - self.textline_light = textline_light self.pcgts = pcgts self.scale_x = None # XXX set outside __init__ self.scale_y = None # XXX set outside __init__ @@ -56,59 +54,56 @@ class EynollahXmlWriter(): points_page_print = points_page_print + ' ' return points_page_print[:-1] - def serialize_lines_in_marginal(self, marginal_region, all_found_textline_polygons_marginals, marginal_idx, page_coord, all_box_coord_marginals, slopes_marginals, counter): - for j in range(len(all_found_textline_polygons_marginals[marginal_idx])): + def serialize_lines_in_marginal(self, marginal_region, all_found_texline_polygons_marginals, marginal_idx, page_coord, all_box_coord_marginals, slopes_marginals, counter): + for j in range(len(all_found_texline_polygons_marginals[marginal_idx])): coords = CoordsType() textline = TextLineType(id=counter.next_line_id, Coords=coords) marginal_region.add_TextLine(textline) - marginal_region.set_orientation(-slopes_marginals[marginal_idx]) + textline.add_TextEquiv(TextEquivType(Unicode='')) points_co = '' - for l in range(len(all_found_textline_polygons_marginals[marginal_idx][j])): - if not (self.curved_line or self.textline_light): - if len(all_found_textline_polygons_marginals[marginal_idx][j][l]) == 2: - textline_x_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x) ) - textline_y_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y) ) + for l in range(len(all_found_texline_polygons_marginals[marginal_idx][j])): + if not self.curved_line: + if len(all_found_texline_polygons_marginals[marginal_idx][j][l]) == 2: + textline_x_coord = max(0, int((all_found_texline_polygons_marginals[marginal_idx][j][l][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x) ) + textline_y_coord = max(0, int((all_found_texline_polygons_marginals[marginal_idx][j][l][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y) ) else: - textline_x_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x) ) - textline_y_coord = max(0, int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y) ) + textline_x_coord = max(0, int((all_found_texline_polygons_marginals[marginal_idx][j][l][0][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x) ) + textline_y_coord = max(0, int((all_found_texline_polygons_marginals[marginal_idx][j][l][0][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y) ) points_co += str(textline_x_coord) points_co += ',' points_co += str(textline_y_coord) - if (self.curved_line or self.textline_light) and np.abs(slopes_marginals[marginal_idx]) <= 45: - if len(all_found_textline_polygons_marginals[marginal_idx][j][l]) == 2: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0] + page_coord[2]) / self.scale_x)) + if self.curved_line and np.abs(slopes_marginals[marginal_idx]) <= 45: + if len(all_found_texline_polygons_marginals[marginal_idx][j][l]) == 2: + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][0] + page_coord[2]) / self.scale_x)) points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][1] + page_coord[0]) / self.scale_y)) + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][1] + page_coord[0]) / self.scale_y)) else: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][0] + page_coord[2]) / self.scale_x)) + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][0][0] + page_coord[2]) / self.scale_x)) points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][1] + page_coord[0]) / self.scale_y)) + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][0][1] + page_coord[0]) / self.scale_y)) - elif (self.curved_line or self.textline_light) and np.abs(slopes_marginals[marginal_idx]) > 45: - if len(all_found_textline_polygons_marginals[marginal_idx][j][l]) == 2: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x)) + elif self.curved_line and np.abs(slopes_marginals[marginal_idx]) > 45: + if len(all_found_texline_polygons_marginals[marginal_idx][j][l]) == 2: + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x)) points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y)) + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y)) else: - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x)) + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][0][0] + all_box_coord_marginals[marginal_idx][2] + page_coord[2]) / self.scale_x)) points_co += ',' - points_co += str(int((all_found_textline_polygons_marginals[marginal_idx][j][l][0][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y)) + points_co += str(int((all_found_texline_polygons_marginals[marginal_idx][j][l][0][1] + all_box_coord_marginals[marginal_idx][0] + page_coord[0]) / self.scale_y)) points_co += ' ' coords.set_points(points_co[:-1]) - def serialize_lines_in_region(self, text_region, all_found_textline_polygons, region_idx, page_coord, all_box_coord, slopes, counter, ocr_all_textlines_textregion): + def serialize_lines_in_region(self, text_region, all_found_texline_polygons, region_idx, page_coord, all_box_coord, slopes, counter): self.logger.debug('enter serialize_lines_in_region') - for j in range(len(all_found_textline_polygons[region_idx])): + for j in range(len(all_found_texline_polygons[region_idx])): coords = CoordsType() - textline = TextLineType(id=counter.next_line_id, Coords=coords) - if ocr_all_textlines_textregion: - textline.set_TextEquiv( [ TextEquivType(Unicode=ocr_all_textlines_textregion[j]) ] ) + textline = TextLineType(id=counter.next_line_id, Coords=coords, TextEquiv=[TextEquivType(index=0, Unicode='')]) text_region.add_TextLine(textline) - text_region.set_orientation(-slopes[region_idx]) region_bboxes = all_box_coord[region_idx] points_co = '' - for idx_contour_textline, contour_textline in enumerate(all_found_textline_polygons[region_idx][j]): - if not (self.curved_line or self.textline_light): + for idx_contour_textline, contour_textline in enumerate(all_found_texline_polygons[region_idx][j]): + if not self.curved_line: if len(contour_textline) == 2: textline_x_coord = max(0, int((contour_textline[0] + region_bboxes[2] + page_coord[2]) / self.scale_x)) textline_y_coord = max(0, int((contour_textline[1] + region_bboxes[0] + page_coord[0]) / self.scale_y)) @@ -119,7 +114,7 @@ class EynollahXmlWriter(): points_co += ',' points_co += str(textline_y_coord) - if (self.curved_line or self.textline_light) and np.abs(slopes[region_idx]) <= 45: + if self.curved_line and np.abs(slopes[region_idx]) <= 45: if len(contour_textline) == 2: points_co += str(int((contour_textline[0] + page_coord[2]) / self.scale_x)) points_co += ',' @@ -128,7 +123,7 @@ class EynollahXmlWriter(): points_co += str(int((contour_textline[0][0] + page_coord[2]) / self.scale_x)) points_co += ',' points_co += str(int((contour_textline[0][1] + page_coord[0])/self.scale_y)) - elif (self.curved_line or self.textline_light) and np.abs(slopes[region_idx]) > 45: + elif self.curved_line and np.abs(slopes[region_idx]) > 45: if len(contour_textline)==2: points_co += str(int((contour_textline[0] + region_bboxes[2] + page_coord[2])/self.scale_x)) points_co += ',' @@ -140,35 +135,13 @@ class EynollahXmlWriter(): points_co += ' ' coords.set_points(points_co[:-1]) - def serialize_lines_in_dropcapital(self, text_region, all_found_textline_polygons, region_idx, page_coord, all_box_coord, slopes, counter, ocr_all_textlines_textregion): - self.logger.debug('enter serialize_lines_in_region') - for j in range(1): - coords = CoordsType() - textline = TextLineType(id=counter.next_line_id, Coords=coords) - if ocr_all_textlines_textregion: - textline.set_TextEquiv( [ TextEquivType(Unicode=ocr_all_textlines_textregion[j]) ] ) - text_region.add_TextLine(textline) - #region_bboxes = all_box_coord[region_idx] - points_co = '' - for idx_contour_textline, contour_textline in enumerate(all_found_textline_polygons[j]): - if len(contour_textline) == 2: - points_co += str(int((contour_textline[0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[1] + page_coord[0]) / self.scale_y)) - else: - points_co += str(int((contour_textline[0][0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((contour_textline[0][1] + page_coord[0])/self.scale_y)) - - points_co += ' ' - coords.set_points(points_co[:-1]) - def write_pagexml(self, pcgts): - self.logger.info("output filename: '%s'", self.output_filename) - with open(self.output_filename, 'w') as f: + out_fname = os.path.join(self.dir_out, self.image_filename_stem) + ".xml" + self.logger.info("output filename: '%s'", out_fname) + with open(out_fname, 'w') as f: f.write(to_xml(pcgts)) - def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, found_polygons_tables, ocr_all_textlines, conf_contours_textregion): + def build_pagexml_no_full_layout(self, found_polygons_text_region, page_coord, order_of_texts, id_of_texts, all_found_texline_polygons, all_box_coord, found_polygons_text_region_img, found_polygons_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml): self.logger.debug('enter build_pagexml_no_full_layout') # create the file structure @@ -184,41 +157,28 @@ class EynollahXmlWriter(): for mm in range(len(found_polygons_text_region)): textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord), conf=conf_contours_textregion[mm]), - ) - #textregion.set_conf(conf_contours_textregion[mm]) + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord)), + TextEquiv=[TextEquivType(index=0, Unicode='')]) page.add_TextRegion(textregion) - if ocr_all_textlines: - ocr_textlines = ocr_all_textlines[mm] - else: - ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, all_box_coord, slopes, counter, ocr_textlines) + self.serialize_lines_in_region(textregion, all_found_texline_polygons, mm, page_coord, all_box_coord, slopes, counter) for mm in range(len(found_polygons_marginals)): marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) page.add_TextRegion(marginal) - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) + self.serialize_lines_in_marginal(marginal, all_found_texline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) for mm in range(len(found_polygons_text_region_img)): img_region = ImageRegionType(id=counter.next_region_id, Coords=CoordsType()) page.add_ImageRegion(img_region) points_co = '' for lmm in range(len(found_polygons_text_region_img[mm])): - try: - points_co += str(int((found_polygons_text_region_img[mm][lmm,0,0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((found_polygons_text_region_img[mm][lmm,0,1] + page_coord[0]) / self.scale_y)) - points_co += ' ' - except: - - points_co += str(int((found_polygons_text_region_img[mm][lmm][0] + page_coord[2])/ self.scale_x )) - points_co += ',' - points_co += str(int((found_polygons_text_region_img[mm][lmm][1] + page_coord[0])/ self.scale_y )) - points_co += ' ' - + points_co += str(int((found_polygons_text_region_img[mm][lmm,0,0] + page_coord[2]) / self.scale_x)) + points_co += ',' + points_co += str(int((found_polygons_text_region_img[mm][lmm,0,1] + page_coord[0]) / self.scale_y)) + points_co += ' ' img_region.get_Coords().set_points(points_co[:-1]) - + for mm in range(len(polygons_lines_to_be_written_in_xml)): sep_hor = SeparatorRegionType(id=counter.next_region_id, Coords=CoordsType()) page.add_SeparatorRegion(sep_hor) @@ -229,20 +189,10 @@ class EynollahXmlWriter(): points_co += str(int((polygons_lines_to_be_written_in_xml[mm][lmm,0,1] ) / self.scale_y)) points_co += ' ' sep_hor.get_Coords().set_points(points_co[:-1]) - for mm in range(len(found_polygons_tables)): - tab_region = TableRegionType(id=counter.next_region_id, Coords=CoordsType()) - page.add_TableRegion(tab_region) - points_co = '' - for lmm in range(len(found_polygons_tables[mm])): - points_co += str(int((found_polygons_tables[mm][lmm,0,0] + page_coord[2]) / self.scale_x)) - points_co += ',' - points_co += str(int((found_polygons_tables[mm][lmm,0,1] + page_coord[0]) / self.scale_y)) - points_co += ' ' - tab_region.get_Coords().set_points(points_co[:-1]) return pcgts - def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals, all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml, ocr_all_textlines, conf_contours_textregion, conf_contours_textregion_h): + def build_pagexml_full_layout(self, found_polygons_text_region, found_polygons_text_region_h, page_coord, order_of_texts, id_of_texts, all_found_texline_polygons, all_found_texline_polygons_h, all_box_coord, all_box_coord_h, found_polygons_text_region_img, found_polygons_tables, found_polygons_drop_capitals, found_polygons_marginals, all_found_texline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, cont_page, polygons_lines_to_be_written_in_xml): self.logger.debug('enter build_pagexml_full_layout') # create the file structure @@ -257,47 +207,37 @@ class EynollahXmlWriter(): for mm in range(len(found_polygons_text_region)): textregion = TextRegionType(id=counter.next_region_id, type_='paragraph', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord), conf=conf_contours_textregion[mm])) + TextEquiv=[TextEquivType(index=0, Unicode='')], + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region[mm], page_coord))) page.add_TextRegion(textregion) - - if ocr_all_textlines: - ocr_textlines = ocr_all_textlines[mm] - else: - ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons, mm, page_coord, all_box_coord, slopes, counter, ocr_textlines) + self.serialize_lines_in_region(textregion, all_found_texline_polygons, mm, page_coord, all_box_coord, slopes, counter) self.logger.debug('len(found_polygons_text_region_h) %s', len(found_polygons_text_region_h)) for mm in range(len(found_polygons_text_region_h)): textregion = TextRegionType(id=counter.next_region_id, type_='header', + TextEquiv=[TextEquivType(index=0, Unicode='')], Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_h[mm], page_coord))) page.add_TextRegion(textregion) - - if ocr_all_textlines: - ocr_textlines = ocr_all_textlines[mm] - else: - ocr_textlines = None - self.serialize_lines_in_region(textregion, all_found_textline_polygons_h, mm, page_coord, all_box_coord_h, slopes_h, counter, ocr_textlines) + self.serialize_lines_in_region(textregion, all_found_texline_polygons_h, mm, page_coord, all_box_coord_h, slopes_h, counter) for mm in range(len(found_polygons_marginals)): marginal = TextRegionType(id=counter.next_region_id, type_='marginalia', + TextEquiv=[TextEquivType(index=0, Unicode='')], Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_marginals[mm], page_coord))) page.add_TextRegion(marginal) - self.serialize_lines_in_marginal(marginal, all_found_textline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) + self.serialize_lines_in_marginal(marginal, all_found_texline_polygons_marginals, mm, page_coord, all_box_coord_marginals, slopes_marginals, counter) for mm in range(len(found_polygons_drop_capitals)): - dropcapital = TextRegionType(id=counter.next_region_id, type_='drop-capital', - Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_drop_capitals[mm], page_coord))) - page.add_TextRegion(dropcapital) - ###all_box_coord_drop = None - ###slopes_drop = None - ###self.serialize_lines_in_dropcapital(dropcapital, [found_polygons_drop_capitals[mm]], mm, page_coord, all_box_coord_drop, slopes_drop, counter, ocr_all_textlines_textregion=None) + page.add_TextRegion(TextRegionType(id=counter.next_region_id, type_='drop-capital', + TextEquiv=[TextEquivType(index=0, Unicode='')], + Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_drop_capitals[mm], page_coord)))) for mm in range(len(found_polygons_text_region_img)): page.add_ImageRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_text_region_img[mm], page_coord)))) - + for mm in range(len(polygons_lines_to_be_written_in_xml)): page.add_SeparatorRegion(ImageRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(polygons_lines_to_be_written_in_xml[mm], [0 , 0, 0, 0])))) - + for mm in range(len(found_polygons_tables)): page.add_TableRegion(TableRegionType(id=counter.next_region_id, Coords=CoordsType(points=self.calculate_polygon_coords(found_polygons_tables[mm], page_coord)))) diff --git a/requirements-test.txt b/requirements-test.txt index cce9428..9bb2a15 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,2 @@ pytest -pytest-subtests -coverage[toml] black diff --git a/requirements.txt b/requirements.txt index 9ed0584..8520780 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ # ocrd includes opencv, numpy, shapely, click -ocrd >= 3.3.0 -numpy <1.24.0 +ocrd >= 2.23.3 +keras >= 2.3.1, < 2.4 scikit-learn >= 0.23.2 -tensorflow < 2.13 -numba <= 0.58.1 -loky +tensorflow-gpu >= 1.15, < 2 +imutils >= 0.5.3 +matplotlib +setuptools >= 50 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9abf158 --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +from setuptools import setup, find_packages +from json import load + +install_requires = open('requirements.txt').read().split('\n') +with open('ocrd-tool.json', 'r', encoding='utf-8') as f: + version = load(f)['version'] + +setup( + name='eynollah', + version=version, + long_description=open('README.md').read(), + long_description_content_type='text/markdown', + author='Vahid Rezanezhad', + url='https://github.com/qurator-spk/eynollah', + license='Apache License 2.0', + namespace_packages=['qurator'], + packages=find_packages(exclude=['tests']), + install_requires=install_requires, + package_data={ + '': ['*.json'] + }, + entry_points={ + 'console_scripts': [ + 'eynollah=qurator.eynollah.cli:main', + 'ocrd-eynollah-segment=qurator.eynollah.ocrd_cli:main', + ] + }, +) diff --git a/src/eynollah/cli.py b/src/eynollah/cli.py deleted file mode 100644 index c189aca..0000000 --- a/src/eynollah/cli.py +++ /dev/null @@ -1,404 +0,0 @@ -import sys -import click -from ocrd_utils import initLogging, getLevelName, getLogger -from eynollah.eynollah import Eynollah, Eynollah_ocr -from eynollah.sbb_binarize import SbbBinarizer - -@click.group() -def main(): - pass - -@main.command() -@click.option( - "--dir_xml", - "-dx", - help="directory of GT page-xml files", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--dir_out_modal_image", - "-domi", - help="directory where ground truth images would be written", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--dir_out_classes", - "-docl", - help="directory where ground truth classes would be written", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--input_height", - "-ih", - help="input height", -) -@click.option( - "--input_width", - "-iw", - help="input width", -) -@click.option( - "--min_area_size", - "-min", - help="min area size of regions considered for reading order training.", -) -def machine_based_reading_order(dir_xml, dir_out_modal_image, dir_out_classes, input_height, input_width, min_area_size): - xml_files_ind = os.listdir(dir_xml) - -@main.command() -@click.option('--patches/--no-patches', default=True, help='by enabling this parameter you let the model to see the image in patches.') -@click.option('--model_dir', '-m', type=click.Path(exists=True, file_okay=False), required=True, help='directory containing models for prediction') -@click.argument('input_image', required=False) -@click.argument('output_image', required=False) -@click.option( - "--dir_in", - "-di", - help="directory of input images", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--dir_out", - "-do", - help="directory for output images", - type=click.Path(exists=True, file_okay=False), -) -def binarization(patches, model_dir, input_image, output_image, dir_in, dir_out): - assert (dir_out is None) == (dir_in is None), "Options -di and -do are mutually dependent" - assert (input_image is None) == (output_image is None), "INPUT_IMAGE and OUTPUT_IMAGE are mutually dependent" - assert (dir_in is None) != (input_image is None), "Specify either -di and -do options, or INPUT_IMAGE and OUTPUT_IMAGE" - SbbBinarizer(model_dir).run(image_path=input_image, use_patches=patches, save=output_image, dir_in=dir_in, dir_out=dir_out) - - - - -@main.command() -@click.option( - "--image", - "-i", - help="image filename", - type=click.Path(exists=True, dir_okay=False), -) - -@click.option( - "--out", - "-o", - help="directory to write output xml data", - type=click.Path(exists=True, file_okay=False), - required=True, -) -@click.option( - "--overwrite", - "-O", - help="overwrite (instead of skipping) if output xml exists", - is_flag=True, -) -@click.option( - "--dir_in", - "-di", - help="directory of images", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--model", - "-m", - help="directory of models", - type=click.Path(exists=True, file_okay=False), - required=True, -) -@click.option( - "--save_images", - "-si", - help="if a directory is given, images in documents will be cropped and saved there", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--save_layout", - "-sl", - help="if a directory is given, plot of layout will be saved there", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--save_deskewed", - "-sd", - help="if a directory is given, deskewed image will be saved there", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--save_all", - "-sa", - help="if a directory is given, all plots needed for documentation will be saved there", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--save_page", - "-sp", - help="if a directory is given, page crop of image will be saved there", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--enable-plotting/--disable-plotting", - "-ep/-noep", - is_flag=True, - help="If set, will plot intermediary files and images", -) -@click.option( - "--extract_only_images/--disable-extracting_only_images", - "-eoi/-noeoi", - is_flag=True, - help="If a directory is given, only images in documents will be cropped and saved there and the other processing will not be done", -) -@click.option( - "--allow-enhancement/--no-allow-enhancement", - "-ae/-noae", - is_flag=True, - help="if this parameter set to true, this tool would check that input image need resizing and enhancement or not. If so output of resized and enhanced image and corresponding layout data will be written in out directory", -) -@click.option( - "--curved-line/--no-curvedline", - "-cl/-nocl", - is_flag=True, - help="if this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline. This should be taken into account that with this option the tool need more time to do process.", -) -@click.option( - "--textline_light/--no-textline_light", - "-tll/-notll", - is_flag=True, - help="if this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline with a faster method.", -) -@click.option( - "--full-layout/--no-full-layout", - "-fl/-nofl", - is_flag=True, - help="if this parameter set to true, this tool will try to return all elements of layout.", -) -@click.option( - "--tables/--no-tables", - "-tab/-notab", - is_flag=True, - help="if this parameter set to true, this tool will try to detect tables.", -) -@click.option( - "--right2left/--left2right", - "-r2l/-l2r", - is_flag=True, - help="if this parameter set to true, this tool will extract right-to-left reading order.", -) -@click.option( - "--input_binary/--input-RGB", - "-ib/-irgb", - is_flag=True, - help="in general, eynollah uses RGB as input but if the input document is strongly dark, bright or for any other reason you can turn binarized input on. This option does not mean that you have to provide a binary image, otherwise this means that the tool itself will binarized the RGB input document.", -) -@click.option( - "--allow_scaling/--no-allow-scaling", - "-as/-noas", - is_flag=True, - help="if this parameter set to true, this tool would check the scale and if needed it will scale it to perform better layout detection", -) -@click.option( - "--headers_off/--headers-on", - "-ho/-noho", - is_flag=True, - help="if this parameter set to true, this tool would ignore headers role in reading order", -) -@click.option( - "--light_version/--original", - "-light/-org", - is_flag=True, - help="if this parameter set to true, this tool would use lighter version", -) -@click.option( - "--ignore_page_extraction/--extract_page_included", - "-ipe/-epi", - is_flag=True, - help="if this parameter set to true, this tool would ignore page extraction", -) -@click.option( - "--reading_order_machine_based/--heuristic_reading_order", - "-romb/-hro", - is_flag=True, - help="if this parameter set to true, this tool would apply machine based reading order detection", -) -@click.option( - "--do_ocr", - "-ocr/-noocr", - is_flag=True, - help="if this parameter set to true, this tool will try to do ocr", -) -@click.option( - "--num_col_upper", - "-ncu", - help="lower limit of columns in document image", -) -@click.option( - "--num_col_lower", - "-ncl", - help="upper limit of columns in document image", -) -@click.option( - "--skip_layout_and_reading_order", - "-slro/-noslro", - is_flag=True, - help="if this parameter set to true, this tool will ignore layout detection and reading order. It means that textline detection will be done within printspace and contours of textline will be written in xml output file.", -) -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", -) - -def layout(image, out, overwrite, dir_in, model, save_images, save_layout, save_deskewed, save_all, extract_only_images, save_page, enable_plotting, allow_enhancement, curved_line, textline_light, full_layout, tables, right2left, input_binary, allow_scaling, headers_off, light_version, reading_order_machine_based, do_ocr, num_col_upper, num_col_lower, skip_layout_and_reading_order, ignore_page_extraction, log_level): - initLogging() - if log_level: - getLogger('eynollah').setLevel(getLevelName(log_level)) - assert enable_plotting or not save_layout, "Plotting with -sl also requires -ep" - assert enable_plotting or not save_deskewed, "Plotting with -sd also requires -ep" - assert enable_plotting or not save_all, "Plotting with -sa also requires -ep" - assert enable_plotting or not save_page, "Plotting with -sp also requires -ep" - assert enable_plotting or not save_images, "Plotting with -si also requires -ep" - assert enable_plotting or not allow_enhancement, "Plotting with -ae also requires -ep" - assert not enable_plotting or save_layout or save_deskewed or save_all or save_page or save_images or allow_enhancement, \ - "Plotting with -ep also requires -sl, -sd, -sa, -sp, -si or -ae" - assert textline_light == light_version, "Both light textline detection -tll and light version -light must be set or unset equally" - assert not extract_only_images or not allow_enhancement, "Image extraction -eoi can not be set alongside allow_enhancement -ae" - assert not extract_only_images or not allow_scaling, "Image extraction -eoi can not be set alongside allow_scaling -as" - assert not extract_only_images or not light_version, "Image extraction -eoi can not be set alongside light_version -light" - assert not extract_only_images or not curved_line, "Image extraction -eoi can not be set alongside curved_line -cl" - assert not extract_only_images or not textline_light, "Image extraction -eoi can not be set alongside textline_light -tll" - assert not extract_only_images or not full_layout, "Image extraction -eoi can not be set alongside full_layout -fl" - assert not extract_only_images or not tables, "Image extraction -eoi can not be set alongside tables -tab" - assert not extract_only_images or not right2left, "Image extraction -eoi can not be set alongside right2left -r2l" - assert not extract_only_images or not headers_off, "Image extraction -eoi can not be set alongside headers_off -ho" - assert image or dir_in, "Either a single image -i or a dir_in -di is required" - eynollah = Eynollah( - model, - logger=getLogger('eynollah'), - dir_out=out, - dir_of_cropped_images=save_images, - extract_only_images=extract_only_images, - dir_of_layout=save_layout, - dir_of_deskewed=save_deskewed, - dir_of_all=save_all, - dir_save_page=save_page, - enable_plotting=enable_plotting, - allow_enhancement=allow_enhancement, - curved_line=curved_line, - textline_light=textline_light, - full_layout=full_layout, - tables=tables, - right2left=right2left, - input_binary=input_binary, - allow_scaling=allow_scaling, - headers_off=headers_off, - light_version=light_version, - ignore_page_extraction=ignore_page_extraction, - reading_order_machine_based=reading_order_machine_based, - do_ocr=do_ocr, - num_col_upper=num_col_upper, - num_col_lower=num_col_lower, - skip_layout_and_reading_order=skip_layout_and_reading_order, - ) - if dir_in: - eynollah.run(dir_in=dir_in, overwrite=overwrite) - else: - eynollah.run(image_filename=image, overwrite=overwrite) - - -@main.command() -@click.option( - "--dir_in", - "-di", - help="directory of images", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--dir_in_bin", - "-dib", - help="directory of binarized images. This should be given if you want to do prediction based on both rgb and bin images. And all bin images are png files", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--out", - "-o", - help="directory to write output xml data", - type=click.Path(exists=True, file_okay=False), - required=True, -) -@click.option( - "--dir_xmls", - "-dx", - help="directory of xmls", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--dir_out_image_text", - "-doit", - help="directory of images with predicted text", - type=click.Path(exists=True, file_okay=False), -) -@click.option( - "--model", - "-m", - help="directory of models", - type=click.Path(exists=True, file_okay=False), - required=True, -) -@click.option( - "--tr_ocr", - "-trocr/-notrocr", - is_flag=True, - help="if this parameter set to true, transformer ocr will be applied, otherwise cnn_rnn model.", -) -@click.option( - "--export_textline_images_and_text", - "-etit/-noetit", - is_flag=True, - help="if this parameter set to true, images and text in xml will be exported into output dir. This files can be used for training a OCR engine.", -) -@click.option( - "--do_not_mask_with_textline_contour", - "-nmtc/-mtc", - is_flag=True, - help="if this parameter set to true, cropped textline images will not be masked with textline contour.", -) -@click.option( - "--draw_texts_on_image", - "-dtoi/-ndtoi", - is_flag=True, - help="if this parameter set to true, the predicted texts will be displayed on an image.", -) -@click.option( - "--prediction_with_both_of_rgb_and_bin", - "-brb/-nbrb", - is_flag=True, - help="If this parameter is set to True, the prediction will be performed using both RGB and binary images. However, this does not necessarily improve results; it may be beneficial for certain document images.", -) -@click.option( - "--log_level", - "-l", - type=click.Choice(['OFF', 'DEBUG', 'INFO', 'WARN', 'ERROR']), - help="Override log level globally to this", -) - -def ocr(dir_in, dir_in_bin, out, dir_xmls, dir_out_image_text, model, tr_ocr, export_textline_images_and_text, do_not_mask_with_textline_contour, draw_texts_on_image, prediction_with_both_of_rgb_and_bin, log_level): - initLogging() - if log_level: - getLogger('eynollah').setLevel(getLevelName(log_level)) - eynollah_ocr = Eynollah_ocr( - dir_xmls=dir_xmls, - dir_out_image_text=dir_out_image_text, - dir_in=dir_in, - dir_in_bin=dir_in_bin, - dir_out=out, - dir_models=model, - tr_ocr=tr_ocr, - export_textline_images_and_text=export_textline_images_and_text, - do_not_mask_with_textline_contour=do_not_mask_with_textline_contour, - draw_texts_on_image=draw_texts_on_image, - prediction_with_both_of_rgb_and_bin=prediction_with_both_of_rgb_and_bin, - ) - eynollah_ocr.run() - -if __name__ == "__main__": - main() diff --git a/src/eynollah/eynollah.py b/src/eynollah/eynollah.py deleted file mode 100644 index d47016b..0000000 --- a/src/eynollah/eynollah.py +++ /dev/null @@ -1,5457 +0,0 @@ -# pylint: disable=no-member,invalid-name,line-too-long,missing-function-docstring,missing-class-docstring,too-many-branches -# pylint: disable=too-many-locals,wrong-import-position,too-many-lines,too-many-statements,chained-comparison,fixme,broad-except,c-extension-no-member -# pylint: disable=too-many-public-methods,too-many-arguments,too-many-instance-attributes,too-many-public-methods, -# pylint: disable=consider-using-enumerate -""" -document layout analysis (segmentation) with output in PAGE-XML -""" - -from logging import Logger -from difflib import SequenceMatcher as sq -from PIL import Image, ImageDraw, ImageFont -import math -import os -import sys -import time -from typing import Optional -import atexit -import warnings -from functools import partial -from pathlib import Path -from multiprocessing import cpu_count -import gc -import copy -import json - -from loky import ProcessPoolExecutor -import xml.etree.ElementTree as ET -import cv2 -import numpy as np -from scipy.signal import find_peaks -from scipy.ndimage import gaussian_filter1d -from numba import cuda - -from ocrd import OcrdPage -from ocrd_utils import getLogger, tf_disable_interactive_logs - -try: - import torch -except ImportError: - torch = None -try: - import matplotlib.pyplot as plt -except ImportError: - plt = None -try: - from transformers import TrOCRProcessor, VisionEncoderDecoderModel -except ImportError: - TrOCRProcessor = VisionEncoderDecoderModel = None - -#os.environ['CUDA_VISIBLE_DEVICES'] = '-1' -tf_disable_interactive_logs() -import tensorflow as tf -from tensorflow.python.keras import backend as K -from tensorflow.keras.models import load_model -tf.get_logger().setLevel("ERROR") -warnings.filterwarnings("ignore") -# use tf1 compatibility for keras backend -from tensorflow.compat.v1.keras.backend import set_session -from tensorflow.keras import layers -from tensorflow.keras.layers import StringLookup - -from .utils.contour import ( - filter_contours_area_of_image, - filter_contours_area_of_image_tables, - find_contours_mean_y_diff, - find_new_features_of_contours, - find_features_of_contours, - get_text_region_boxes_by_given_contours, - get_textregion_contours_in_org_image, - get_textregion_contours_in_org_image_light, - return_contours_of_image, - return_contours_of_interested_region, - return_contours_of_interested_region_by_min_size, - return_contours_of_interested_textline, - return_parent_contours, -) -from .utils.rotate import ( - rotate_image, - rotation_not_90_func, - rotation_not_90_func_full_layout -) -from .utils.separate_lines import ( - textline_contours_postprocessing, - separate_lines_new2, - return_deskew_slop, - do_work_of_slopes_new, - do_work_of_slopes_new_curved, - do_work_of_slopes_new_light, -) -from .utils.drop_capitals import ( - adhere_drop_capital_region_into_corresponding_textline, - filter_small_drop_capitals_from_no_patch_layout -) -from .utils.marginals import get_marginals -from .utils.resize import resize_image -from .utils import ( - boosting_headers_by_longshot_region_segmentation, - crop_image_inside_box, - find_num_col, - otsu_copy_binary, - put_drop_out_from_only_drop_model, - putt_bb_of_drop_capitals_of_model_in_patches_in_layout, - check_any_text_region_in_model_one_is_main_or_header, - check_any_text_region_in_model_one_is_main_or_header_light, - small_textlines_to_parent_adherence2, - order_of_regions, - find_number_of_columns_in_document, - return_boxes_of_images_by_order_of_reading_new -) -from .utils.pil_cv2 import check_dpi, pil2cv -from .utils.xml import order_and_id_of_texts -from .plot import EynollahPlotter -from .writer import EynollahXmlWriter - -MIN_AREA_REGION = 0.000001 -SLOPE_THRESHOLD = 0.13 -RATIO_OF_TWO_MODEL_THRESHOLD = 95.50 #98.45: -DPI_THRESHOLD = 298 -MAX_SLOPE = 999 -KERNEL = np.ones((5, 5), np.uint8) - -projection_dim = 64 -patch_size = 1 -num_patches =21*21#14*14#28*28#14*14#28*28 - - -class Patches(layers.Layer): - def __init__(self, **kwargs): - super(Patches, self).__init__() - self.patch_size = patch_size - - def call(self, images): - batch_size = tf.shape(images)[0] - patches = tf.image.extract_patches( - images=images, - sizes=[1, self.patch_size, self.patch_size, 1], - strides=[1, self.patch_size, self.patch_size, 1], - rates=[1, 1, 1, 1], - padding="VALID", - ) - patch_dims = patches.shape[-1] - patches = tf.reshape(patches, [batch_size, -1, patch_dims]) - return patches - def get_config(self): - - config = super().get_config().copy() - config.update({ - 'patch_size': self.patch_size, - }) - return config - - -class PatchEncoder(layers.Layer): - def __init__(self, **kwargs): - super(PatchEncoder, self).__init__() - self.num_patches = num_patches - self.projection = layers.Dense(units=projection_dim) - self.position_embedding = layers.Embedding( - input_dim=num_patches, output_dim=projection_dim - ) - - def call(self, patch): - positions = tf.range(start=0, limit=self.num_patches, delta=1) - encoded = self.projection(patch) + self.position_embedding(positions) - return encoded - def get_config(self): - - config = super().get_config().copy() - config.update({ - 'num_patches': self.num_patches, - 'projection': self.projection, - 'position_embedding': self.position_embedding, - }) - return config - -class Eynollah: - def __init__( - self, - dir_models : str, - dir_out : Optional[str] = None, - dir_of_cropped_images : Optional[str] = None, - extract_only_images : bool =False, - dir_of_layout : Optional[str] = None, - dir_of_deskewed : Optional[str] = None, - dir_of_all : Optional[str] = None, - dir_save_page : Optional[str] = None, - enable_plotting : bool = False, - allow_enhancement : bool = False, - curved_line : bool = False, - textline_light : bool = False, - full_layout : bool = False, - tables : bool = False, - right2left : bool = False, - input_binary : bool = False, - allow_scaling : bool = False, - headers_off : bool = False, - light_version : bool = False, - ignore_page_extraction : bool = False, - reading_order_machine_based : bool = False, - do_ocr : bool = False, - num_col_upper : Optional[int] = None, - num_col_lower : Optional[int] = None, - skip_layout_and_reading_order : bool = False, - logger : Optional[Logger] = None, - ): - if skip_layout_and_reading_order: - textline_light = True - self.light_version = light_version - self.dir_out = dir_out - self.dir_of_all = dir_of_all - self.dir_save_page = dir_save_page - self.reading_order_machine_based = reading_order_machine_based - self.dir_of_deskewed = dir_of_deskewed - self.dir_of_deskewed = dir_of_deskewed - self.dir_of_cropped_images=dir_of_cropped_images - self.dir_of_layout=dir_of_layout - self.enable_plotting = enable_plotting - self.allow_enhancement = allow_enhancement - self.curved_line = curved_line - self.textline_light = textline_light - self.full_layout = full_layout - self.tables = tables - self.right2left = right2left - self.input_binary = input_binary - self.allow_scaling = allow_scaling - self.headers_off = headers_off - self.light_version = light_version - self.extract_only_images = extract_only_images - self.ignore_page_extraction = ignore_page_extraction - self.skip_layout_and_reading_order = skip_layout_and_reading_order - self.ocr = do_ocr - if num_col_upper: - self.num_col_upper = int(num_col_upper) - else: - self.num_col_upper = num_col_upper - if num_col_lower: - self.num_col_lower = int(num_col_lower) - else: - self.num_col_lower = num_col_lower - self.logger = logger if logger else getLogger('eynollah') - # for parallelization of CPU-intensive tasks: - self.executor = ProcessPoolExecutor(max_workers=cpu_count(), timeout=1200) - atexit.register(self.executor.shutdown) - self.dir_models = dir_models - self.model_dir_of_enhancement = dir_models + "/eynollah-enhancement_20210425" - self.model_dir_of_binarization = dir_models + "/eynollah-binarization_20210425" - self.model_dir_of_col_classifier = dir_models + "/eynollah-column-classifier_20210425" - self.model_region_dir_p = dir_models + "/eynollah-main-regions-aug-scaling_20210425" - self.model_region_dir_p2 = dir_models + "/eynollah-main-regions-aug-rotation_20210425" - #"/modelens_full_lay_1_3_031124" - #"/modelens_full_lay_13__3_19_241024" - #"/model_full_lay_13_241024" - #"/modelens_full_lay_13_17_231024" - #"/modelens_full_lay_1_2_221024" - #"/eynollah-full-regions-1column_20210425" - self.model_region_dir_fully_np = dir_models + "/modelens_full_lay_1__4_3_091124" - #self.model_region_dir_fully = dir_models + "/eynollah-full-regions-3+column_20210425" - self.model_page_dir = dir_models + "/eynollah-page-extraction_20210425" - self.model_region_dir_p_ens = dir_models + "/eynollah-main-regions-ensembled_20210425" - self.model_region_dir_p_ens_light = dir_models + "/eynollah-main-regions_20220314" - self.model_region_dir_p_ens_light_only_images_extraction = dir_models + "/eynollah-main-regions_20231127_672_org_ens_11_13_16_17_18" - self.model_reading_order_dir = dir_models + "/model_ens_reading_order_machine_based" - #"/modelens_12sp_elay_0_3_4__3_6_n" - #"/modelens_earlylayout_12spaltige_2_3_5_6_7_8" - #"/modelens_early12_sp_2_3_5_6_7_8_9_10_12_14_15_16_18" - #"/modelens_1_2_4_5_early_lay_1_2_spaltige" - #"/model_3_eraly_layout_no_patches_1_2_spaltige" - self.model_region_dir_p_1_2_sp_np = dir_models + "/modelens_e_l_all_sp_0_1_2_3_4_171024" - ##self.model_region_dir_fully_new = dir_models + "/model_2_full_layout_new_trans" - #"/modelens_full_lay_1_3_031124" - #"/modelens_full_lay_13__3_19_241024" - #"/model_full_lay_13_241024" - #"/modelens_full_lay_13_17_231024" - #"/modelens_full_lay_1_2_221024" - #"/modelens_full_layout_24_till_28" - #"/model_2_full_layout_new_trans" - self.model_region_dir_fully = dir_models + "/modelens_full_lay_1__4_3_091124" - if self.textline_light: - #"/modelens_textline_1_4_16092024" - #"/model_textline_ens_3_4_5_6_artificial" - #"/modelens_textline_1_3_4_20240915" - #"/model_textline_ens_3_4_5_6_artificial" - #"/modelens_textline_9_12_13_14_15" - #"/eynollah-textline_light_20210425" - self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024" - else: - #"/eynollah-textline_20210425" - self.model_textline_dir = dir_models + "/modelens_textline_0_1__2_4_16092024" - if self.ocr: - self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" - if self.tables: - if self.light_version: - self.model_table_dir = dir_models + "/modelens_table_0t4_201124" - else: - self.model_table_dir = dir_models + "/eynollah-tables_20210319" - - # #gpu_options = tf.compat.v1.GPUOptions(allow_growth=True) - # #gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=7.7, allow_growth=True) - # #session = tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(gpu_options=gpu_options)) - # config = tf.compat.v1.ConfigProto() - # config.gpu_options.allow_growth = True - # #session = tf.InteractiveSession() - # session = tf.compat.v1.Session(config=config) - # set_session(session) - try: - for device in tf.config.list_physical_devices('GPU'): - tf.config.experimental.set_memory_growth(device, True) - except: - self.logger.warning("no GPU device available") - - self.model_page = self.our_load_model(self.model_page_dir) - self.model_classifier = self.our_load_model(self.model_dir_of_col_classifier) - self.model_bin = self.our_load_model(self.model_dir_of_binarization) - if self.extract_only_images: - self.model_region = self.our_load_model(self.model_region_dir_p_ens_light_only_images_extraction) - else: - self.model_textline = self.our_load_model(self.model_textline_dir) - if self.light_version: - self.model_region = self.our_load_model(self.model_region_dir_p_ens_light) - self.model_region_1_2 = self.our_load_model(self.model_region_dir_p_1_2_sp_np) - else: - self.model_region = self.our_load_model(self.model_region_dir_p_ens) - self.model_region_p2 = self.our_load_model(self.model_region_dir_p2) - self.model_enhancement = self.our_load_model(self.model_dir_of_enhancement) - ###self.model_region_fl_new = self.our_load_model(self.model_region_dir_fully_new) - self.model_region_fl_np = self.our_load_model(self.model_region_dir_fully_np) - self.model_region_fl = self.our_load_model(self.model_region_dir_fully) - if self.reading_order_machine_based: - self.model_reading_order = self.our_load_model(self.model_reading_order_dir) - if self.ocr: - self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - #("microsoft/trocr-base-printed")#("microsoft/trocr-base-handwritten") - self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-handwritten") - if self.tables: - self.model_table = self.our_load_model(self.model_table_dir) - - def cache_images(self, image_filename=None, image_pil=None, dpi=None): - ret = {} - t_c0 = time.time() - if image_filename: - ret['img'] = cv2.imread(image_filename) - if self.light_version: - self.dpi = 100 - else: - self.dpi = check_dpi(image_filename) - else: - ret['img'] = pil2cv(image_pil) - if self.light_version: - self.dpi = 100 - else: - self.dpi = check_dpi(image_pil) - ret['img_grayscale'] = cv2.cvtColor(ret['img'], cv2.COLOR_BGR2GRAY) - for prefix in ('', '_grayscale'): - ret[f'img{prefix}_uint8'] = ret[f'img{prefix}'].astype(np.uint8) - self._imgs = ret - if dpi is not None: - self.dpi = dpi - - def reset_file_name_dir(self, image_filename): - t_c = time.time() - self.cache_images(image_filename=image_filename) - - self.plotter = None if not self.enable_plotting else EynollahPlotter( - dir_out=self.dir_out, - dir_of_all=self.dir_of_all, - dir_save_page=self.dir_save_page, - dir_of_deskewed=self.dir_of_deskewed, - dir_of_cropped_images=self.dir_of_cropped_images, - dir_of_layout=self.dir_of_layout, - image_filename_stem=Path(Path(image_filename).name).stem) - - self.writer = EynollahXmlWriter( - dir_out=self.dir_out, - image_filename=image_filename, - curved_line=self.curved_line, - textline_light = self.textline_light) - - def imread(self, grayscale=False, uint8=True): - key = 'img' - if grayscale: - key += '_grayscale' - if uint8: - key += '_uint8' - return self._imgs[key].copy() - - def isNaN(self, num): - return num != num - - def predict_enhancement(self, img): - self.logger.debug("enter predict_enhancement") - - img_height_model = self.model_enhancement.layers[-1].output_shape[1] - img_width_model = self.model_enhancement.layers[-1].output_shape[2] - if img.shape[0] < img_height_model: - img = cv2.resize(img, (img.shape[1], img_width_model), interpolation=cv2.INTER_NEAREST) - if img.shape[1] < img_width_model: - img = cv2.resize(img, (img_height_model, img.shape[0]), interpolation=cv2.INTER_NEAREST) - margin = int(0 * img_width_model) - width_mid = img_width_model - 2 * margin - height_mid = img_height_model - 2 * margin - img = img / 255. - img_h = img.shape[0] - img_w = img.shape[1] - - prediction_true = np.zeros((img_h, img_w, 3)) - nxf = img_w / float(width_mid) - nyf = img_h / float(height_mid) - nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) - nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) - - for i in range(nxf): - for j in range(nyf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - else: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - if j == 0: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - else: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - - if index_x_u > img_w: - index_x_u = img_w - index_x_d = img_w - img_width_model - if index_y_u > img_h: - index_y_u = img_h - index_y_d = img_h - img_height_model - - img_patch = img[np.newaxis, index_y_d:index_y_u, index_x_d:index_x_u, :] - label_p_pred = self.model_enhancement.predict(img_patch, verbose=0) - seg = label_p_pred[0, :, :, :] * 255 - - if i == 0 and j == 0: - prediction_true[index_y_d + 0:index_y_u - margin, - index_x_d + 0:index_x_u - margin] = \ - seg[0:-margin or None, - 0:-margin or None] - elif i == nxf - 1 and j == nyf - 1: - prediction_true[index_y_d + margin:index_y_u - 0, - index_x_d + margin:index_x_u - 0] = \ - seg[margin:, - margin:] - elif i == 0 and j == nyf - 1: - prediction_true[index_y_d + margin:index_y_u - 0, - index_x_d + 0:index_x_u - margin] = \ - seg[margin:, - 0:-margin or None] - elif i == nxf - 1 and j == 0: - prediction_true[index_y_d + 0:index_y_u - margin, - index_x_d + margin:index_x_u - 0] = \ - seg[0:-margin or None, - margin:] - elif i == 0 and j != 0 and j != nyf - 1: - prediction_true[index_y_d + margin:index_y_u - margin, - index_x_d + 0:index_x_u - margin] = \ - seg[margin:-margin or None, - 0:-margin or None] - elif i == nxf - 1 and j != 0 and j != nyf - 1: - prediction_true[index_y_d + margin:index_y_u - margin, - index_x_d + margin:index_x_u - 0] = \ - seg[margin:-margin or None, - margin:] - elif i != 0 and i != nxf - 1 and j == 0: - prediction_true[index_y_d + 0:index_y_u - margin, - index_x_d + margin:index_x_u - margin] = \ - seg[0:-margin or None, - margin:-margin or None] - elif i != 0 and i != nxf - 1 and j == nyf - 1: - prediction_true[index_y_d + margin:index_y_u - 0, - index_x_d + margin:index_x_u - margin] = \ - seg[margin:, - margin:-margin or None] - else: - prediction_true[index_y_d + margin:index_y_u - margin, - index_x_d + margin:index_x_u - margin] = \ - seg[margin:-margin or None, - margin:-margin or None] - - prediction_true = prediction_true.astype(int) - return prediction_true - - def calculate_width_height_by_columns(self, img, num_col, width_early, label_p_pred): - self.logger.debug("enter calculate_width_height_by_columns") - if num_col == 1 and width_early < 1100: - img_w_new = 2000 - elif num_col == 1 and width_early >= 2500: - img_w_new = 2000 - elif num_col == 1 and width_early >= 1100 and width_early < 2500: - img_w_new = width_early - elif num_col == 2 and width_early < 2000: - img_w_new = 2400 - elif num_col == 2 and width_early >= 3500: - img_w_new = 2400 - elif num_col == 2 and width_early >= 2000 and width_early < 3500: - img_w_new = width_early - elif num_col == 3 and width_early < 2000: - img_w_new = 3000 - elif num_col == 3 and width_early >= 4000: - img_w_new = 3000 - elif num_col == 3 and width_early >= 2000 and width_early < 4000: - img_w_new = width_early - elif num_col == 4 and width_early < 2500: - img_w_new = 4000 - elif num_col == 4 and width_early >= 5000: - img_w_new = 4000 - elif num_col == 4 and width_early >= 2500 and width_early < 5000: - img_w_new = width_early - elif num_col == 5 and width_early < 3700: - img_w_new = 5000 - elif num_col == 5 and width_early >= 7000: - img_w_new = 5000 - elif num_col == 5 and width_early >= 3700 and width_early < 7000: - img_w_new = width_early - elif num_col == 6 and width_early < 4500: - img_w_new = 6500 # 5400 - else: - img_w_new = width_early - img_h_new = img_w_new * img.shape[0] // img.shape[1] - - if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: - img_new = np.copy(img) - num_column_is_classified = False - #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: - elif img_h_new >= 8000: - img_new = np.copy(img) - num_column_is_classified = False - else: - img_new = resize_image(img, img_h_new, img_w_new) - num_column_is_classified = True - - return img_new, num_column_is_classified - - def calculate_width_height_by_columns_1_2(self, img, num_col, width_early, label_p_pred): - self.logger.debug("enter calculate_width_height_by_columns") - if num_col == 1: - img_w_new = 1000 - else: - img_w_new = 1300 - img_h_new = img_w_new * img.shape[0] // img.shape[1] - - if label_p_pred[0][int(num_col - 1)] < 0.9 and img_w_new < width_early: - img_new = np.copy(img) - num_column_is_classified = False - #elif label_p_pred[0][int(num_col - 1)] < 0.8 and img_h_new >= 8000: - elif img_h_new >= 8000: - img_new = np.copy(img) - num_column_is_classified = False - else: - img_new = resize_image(img, img_h_new, img_w_new) - num_column_is_classified = True - - return img_new, num_column_is_classified - - def calculate_width_height_by_columns_extract_only_images(self, img, num_col, width_early, label_p_pred): - self.logger.debug("enter calculate_width_height_by_columns") - if num_col == 1: - img_w_new = 700 - elif num_col == 2: - img_w_new = 900 - elif num_col == 3: - img_w_new = 1500 - elif num_col == 4: - img_w_new = 1800 - elif num_col == 5: - img_w_new = 2200 - elif num_col == 6: - img_w_new = 2500 - img_h_new = img_w_new * img.shape[0] // img.shape[1] - - img_new = resize_image(img, img_h_new, img_w_new) - num_column_is_classified = True - - return img_new, num_column_is_classified - - def resize_image_with_column_classifier(self, is_image_enhanced, img_bin): - self.logger.debug("enter resize_image_with_column_classifier") - if self.input_binary: - img = np.copy(img_bin) - else: - img = self.imread() - - _, page_coord = self.early_page_for_num_of_column_classification(img) - - if self.input_binary: - img_in = np.copy(img) - img_in = img_in / 255.0 - width_early = img_in.shape[1] - img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = img_in.reshape(1, 448, 448, 3) - else: - img_1ch = self.imread(grayscale=True, uint8=False) - width_early = img_1ch.shape[1] - img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - # plt.imshow(img_1ch) - # plt.show() - img_1ch = img_1ch / 255.0 - img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) - - img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) - img_in[0, :, :, 0] = img_1ch[:, :] - img_in[0, :, :, 1] = img_1ch[:, :] - img_in[0, :, :, 2] = img_1ch[:, :] - - label_p_pred = self.model_classifier.predict(img_in, verbose=0) - num_col = np.argmax(label_p_pred[0]) + 1 - - self.logger.info("Found %s columns (%s)", num_col, label_p_pred) - img_new, _ = self.calculate_width_height_by_columns(img, num_col, width_early, label_p_pred) - - if img_new.shape[1] > img.shape[1]: - img_new = self.predict_enhancement(img_new) - is_image_enhanced = True - - return img, img_new, is_image_enhanced - - def resize_and_enhance_image_with_column_classifier(self, light_version): - self.logger.debug("enter resize_and_enhance_image_with_column_classifier") - dpi = self.dpi - self.logger.info("Detected %s DPI", dpi) - if self.input_binary: - img = self.imread() - prediction_bin = self.do_prediction(True, img, self.model_bin, n_batch_inference=5) - prediction_bin = 255 * (prediction_bin[:,:,0]==0) - prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2).astype(np.uint8) - img= np.copy(prediction_bin) - img_bin = prediction_bin - else: - img = self.imread() - img_bin = None - - width_early = img.shape[1] - t1 = time.time() - _, page_coord = self.early_page_for_num_of_column_classification(img_bin) - - self.image_page_org_size = img[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3], :] - self.page_coord = page_coord - - if self.num_col_upper and not self.num_col_lower: - num_col = self.num_col_upper - label_p_pred = [np.ones(6)] - elif self.num_col_lower and not self.num_col_upper: - num_col = self.num_col_lower - label_p_pred = [np.ones(6)] - elif not self.num_col_upper and not self.num_col_lower: - if self.input_binary: - img_in = np.copy(img) - img_in = img_in / 255.0 - img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = img_in.reshape(1, 448, 448, 3) - else: - img_1ch = self.imread(grayscale=True) - width_early = img_1ch.shape[1] - img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - img_1ch = img_1ch / 255.0 - img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) - img_in[0, :, :, 0] = img_1ch[:, :] - img_in[0, :, :, 1] = img_1ch[:, :] - img_in[0, :, :, 2] = img_1ch[:, :] - - label_p_pred = self.model_classifier.predict(img_in, verbose=0) - num_col = np.argmax(label_p_pred[0]) + 1 - elif (self.num_col_upper and self.num_col_lower) and (self.num_col_upper!=self.num_col_lower): - if self.input_binary: - img_in = np.copy(img) - img_in = img_in / 255.0 - img_in = cv2.resize(img_in, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = img_in.reshape(1, 448, 448, 3) - else: - img_1ch = self.imread(grayscale=True) - width_early = img_1ch.shape[1] - img_1ch = img_1ch[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - img_1ch = img_1ch / 255.0 - img_1ch = cv2.resize(img_1ch, (448, 448), interpolation=cv2.INTER_NEAREST) - img_in = np.zeros((1, img_1ch.shape[0], img_1ch.shape[1], 3)) - img_in[0, :, :, 0] = img_1ch[:, :] - img_in[0, :, :, 1] = img_1ch[:, :] - img_in[0, :, :, 2] = img_1ch[:, :] - - label_p_pred = self.model_classifier.predict(img_in, verbose=0) - num_col = np.argmax(label_p_pred[0]) + 1 - - if num_col > self.num_col_upper: - num_col = self.num_col_upper - label_p_pred = [np.ones(6)] - if num_col < self.num_col_lower: - num_col = self.num_col_lower - label_p_pred = [np.ones(6)] - else: - num_col = self.num_col_upper - label_p_pred = [np.ones(6)] - - self.logger.info("Found %d columns (%s)", num_col, np.around(label_p_pred, decimals=5)) - if not self.extract_only_images: - if dpi < DPI_THRESHOLD: - if light_version and num_col in (1,2): - img_new, num_column_is_classified = self.calculate_width_height_by_columns_1_2( - img, num_col, width_early, label_p_pred) - else: - img_new, num_column_is_classified = self.calculate_width_height_by_columns( - img, num_col, width_early, label_p_pred) - if light_version: - image_res = np.copy(img_new) - else: - image_res = self.predict_enhancement(img_new) - is_image_enhanced = True - else: - if light_version and num_col in (1,2): - img_new, num_column_is_classified = self.calculate_width_height_by_columns_1_2( - img, num_col, width_early, label_p_pred) - image_res = np.copy(img_new) - is_image_enhanced = True - else: - num_column_is_classified = True - image_res = np.copy(img) - is_image_enhanced = False - else: - num_column_is_classified = True - image_res = np.copy(img) - is_image_enhanced = False - - self.logger.debug("exit resize_and_enhance_image_with_column_classifier") - return is_image_enhanced, img, image_res, num_col, num_column_is_classified, img_bin - - # pylint: disable=attribute-defined-outside-init - def get_image_and_scales(self, img_org, img_res, scale): - self.logger.debug("enter get_image_and_scales") - self.image = np.copy(img_res) - self.image_org = np.copy(img_org) - self.height_org = self.image.shape[0] - self.width_org = self.image.shape[1] - - self.img_hight_int = int(self.image.shape[0] * scale) - self.img_width_int = int(self.image.shape[1] * scale) - self.scale_y = self.img_hight_int / float(self.image.shape[0]) - self.scale_x = self.img_width_int / float(self.image.shape[1]) - - self.image = resize_image(self.image, self.img_hight_int, self.img_width_int) - - # Also set for the plotter - if self.plotter: - self.plotter.image_org = self.image_org - self.plotter.scale_y = self.scale_y - self.plotter.scale_x = self.scale_x - # Also set for the writer - self.writer.image_org = self.image_org - self.writer.scale_y = self.scale_y - self.writer.scale_x = self.scale_x - self.writer.height_org = self.height_org - self.writer.width_org = self.width_org - - def get_image_and_scales_after_enhancing(self, img_org, img_res): - self.logger.debug("enter get_image_and_scales_after_enhancing") - self.image = np.copy(img_res) - self.image = self.image.astype(np.uint8) - self.image_org = np.copy(img_org) - self.height_org = self.image_org.shape[0] - self.width_org = self.image_org.shape[1] - - self.scale_y = img_res.shape[0] / float(self.image_org.shape[0]) - self.scale_x = img_res.shape[1] / float(self.image_org.shape[1]) - - # Also set for the plotter - if self.plotter: - self.plotter.image_org = self.image_org - self.plotter.scale_y = self.scale_y - self.plotter.scale_x = self.scale_x - # Also set for the writer - self.writer.image_org = self.image_org - self.writer.scale_y = self.scale_y - self.writer.scale_x = self.scale_x - self.writer.height_org = self.height_org - self.writer.width_org = self.width_org - - def do_prediction( - self, patches, img, model, - n_batch_inference=1, marginal_of_patch_percent=0.1, - thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False): - - self.logger.debug("enter do_prediction") - img_height_model = model.layers[-1].output_shape[1] - img_width_model = model.layers[-1].output_shape[2] - - if not patches: - img_h_page = img.shape[0] - img_w_page = img.shape[1] - img = img / float(255.0) - img = resize_image(img, img_height_model, img_width_model) - - label_p_pred = model.predict(img[np.newaxis], verbose=0) - seg = np.argmax(label_p_pred, axis=3)[0] - - if thresholding_for_artificial_class_in_light_version: - seg_art = label_p_pred[0,:,:,2] - - seg_art[seg_art<0.2] = 0 - seg_art[seg_art>0] =1 - - seg[seg_art==1]=2 - seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) - prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) - return prediction_true - - if img.shape[0] < img_height_model: - img = resize_image(img, img_height_model, img.shape[1]) - if img.shape[1] < img_width_model: - img = resize_image(img, img.shape[0], img_width_model) - - self.logger.debug("Patch size: %sx%s", img_height_model, img_width_model) - margin = int(marginal_of_patch_percent * img_height_model) - width_mid = img_width_model - 2 * margin - height_mid = img_height_model - 2 * margin - img = img / 255. - #img = img.astype(np.float16) - img_h = img.shape[0] - img_w = img.shape[1] - prediction_true = np.zeros((img_h, img_w, 3)) - mask_true = np.zeros((img_h, img_w)) - nxf = img_w / float(width_mid) - nyf = img_h / float(height_mid) - nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) - nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - img_patch = np.zeros((n_batch_inference, img_height_model, img_width_model, 3)) - for i in range(nxf): - for j in range(nyf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - else: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - if j == 0: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - else: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - if index_x_u > img_w: - index_x_u = img_w - index_x_d = img_w - img_width_model - if index_y_u > img_h: - index_y_u = img_h - index_y_d = img_h - img_height_model - - list_i_s.append(i) - list_j_s.append(j) - list_x_u.append(index_x_u) - list_x_d.append(index_x_d) - list_y_d.append(index_y_d) - list_y_u.append(index_y_u) - - img_patch[batch_indexer,:,:,:] = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - batch_indexer += 1 - - if (batch_indexer == n_batch_inference or - # last batch - i == nxf - 1 and j == nyf - 1): - self.logger.debug("predicting patches on %s", str(img_patch.shape)) - label_p_pred = model.predict(img_patch, verbose=0) - seg = np.argmax(label_p_pred, axis=3) - - if thresholding_for_some_classes_in_light_version: - seg_not_base = label_p_pred[:,:,:,4] - seg_not_base[seg_not_base>0.03] =1 - seg_not_base[seg_not_base<1] =0 - - seg_line = label_p_pred[:,:,:,3] - seg_line[seg_line>0.1] =1 - seg_line[seg_line<1] =0 - - seg_background = label_p_pred[:,:,:,0] - seg_background[seg_background>0.25] =1 - seg_background[seg_background<1] =0 - - seg[seg_not_base==1]=4 - seg[seg_background==1]=0 - seg[(seg_line==1) & (seg==0)]=3 - if thresholding_for_artificial_class_in_light_version: - seg_art = label_p_pred[:,:,:,2] - - seg_art[seg_art<0.2] = 0 - seg_art[seg_art>0] =1 - - seg[seg_art==1]=2 - - indexer_inside_batch = 0 - for i_batch, j_batch in zip(list_i_s, list_j_s): - seg_in = seg[indexer_inside_batch] - - index_y_u_in = list_y_u[indexer_inside_batch] - index_y_d_in = list_y_d[indexer_inside_batch] - - index_x_u_in = list_x_u[indexer_inside_batch] - index_x_d_in = list_x_d[indexer_inside_batch] - - if i_batch == 0 and j_batch == 0: - prediction_true[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + 0:index_x_u_in - margin] = \ - seg_in[0:-margin or None, - 0:-margin or None, - np.newaxis] - elif i_batch == nxf - 1 and j_batch == nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + margin:index_x_u_in - 0] = \ - seg_in[margin:, - margin:, - np.newaxis] - elif i_batch == 0 and j_batch == nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + 0:index_x_u_in - margin] = \ - seg_in[margin:, - 0:-margin or None, - np.newaxis] - elif i_batch == nxf - 1 and j_batch == 0: - prediction_true[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - 0] = \ - seg_in[0:-margin or None, - margin:, - np.newaxis] - elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + 0:index_x_u_in - margin] = \ - seg_in[margin:-margin or None, - 0:-margin or None, - np.newaxis] - elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - 0] = \ - seg_in[margin:-margin or None, - margin:, - np.newaxis] - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: - prediction_true[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - margin] = \ - seg_in[0:-margin or None, - margin:-margin or None, - np.newaxis] - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + margin:index_x_u_in - margin] = \ - seg_in[margin:, - margin:-margin or None, - np.newaxis] - else: - prediction_true[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - margin] = \ - seg_in[margin:-margin or None, - margin:-margin or None, - np.newaxis] - indexer_inside_batch += 1 - - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - img_patch[:] = 0 - - prediction_true = prediction_true.astype(np.uint8) - #del model - gc.collect() - return prediction_true - - def do_padding_with_scale(self, img, scale): - h_n = int(img.shape[0]*scale) - w_n = int(img.shape[1]*scale) - - channel0_avg = int( np.mean(img[:,:,0]) ) - channel1_avg = int( np.mean(img[:,:,1]) ) - channel2_avg = int( np.mean(img[:,:,2]) ) - - h_diff = img.shape[0] - h_n - w_diff = img.shape[1] - w_n - - h_start = int(0.5 * h_diff) - w_start = int(0.5 * w_diff) - - img_res = resize_image(img, h_n, w_n) - #label_res = resize_image(label, h_n, w_n) - - img_scaled_padded = np.copy(img) - - #label_scaled_padded = np.zeros(label.shape) - - img_scaled_padded[:,:,0] = channel0_avg - img_scaled_padded[:,:,1] = channel1_avg - img_scaled_padded[:,:,2] = channel2_avg - - img_scaled_padded[h_start:h_start+h_n, w_start:w_start+w_n,:] = img_res[:,:,:] - #label_scaled_padded[h_start:h_start+h_n, w_start:w_start+w_n,:] = label_res[:,:,:] - - return img_scaled_padded#, label_scaled_padded - - def do_prediction_new_concept_scatter_nd( - self, patches, img, model, - n_batch_inference=1, marginal_of_patch_percent=0.1, - thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False): - - self.logger.debug("enter do_prediction_new_concept") - img_height_model = model.layers[-1].output_shape[1] - img_width_model = model.layers[-1].output_shape[2] - - if not patches: - img_h_page = img.shape[0] - img_w_page = img.shape[1] - img = img / 255.0 - img = resize_image(img, img_height_model, img_width_model) - - label_p_pred = model.predict(img[np.newaxis], verbose=0) - seg = np.argmax(label_p_pred, axis=3)[0] - - if thresholding_for_artificial_class_in_light_version: - #seg_text = label_p_pred[0,:,:,1] - #seg_text[seg_text<0.2] =0 - #seg_text[seg_text>0] =1 - #seg[seg_text==1]=1 - - seg_art = label_p_pred[0,:,:,4] - seg_art[seg_art<0.2] =0 - seg_art[seg_art>0] =1 - seg[seg_art==1]=4 - - seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) - prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) - return prediction_true - - if img.shape[0] < img_height_model: - img = resize_image(img, img_height_model, img.shape[1]) - if img.shape[1] < img_width_model: - img = resize_image(img, img.shape[0], img_width_model) - - self.logger.debug("Patch size: %sx%s", img_height_model, img_width_model) - ##margin = int(marginal_of_patch_percent * img_height_model) - #width_mid = img_width_model - 2 * margin - #height_mid = img_height_model - 2 * margin - img = img / 255.0 - img = img.astype(np.float16) - img_h = img.shape[0] - img_w = img.shape[1] - - stride_x = img_width_model - 100 - stride_y = img_height_model - 100 - - one_tensor = tf.ones_like(img) - img_patches, one_patches = tf.image.extract_patches( - images=[img, one_tensor], - sizes=[1, img_height_model, img_width_model, 1], - strides=[1, stride_y, stride_x, 1], - rates=[1, 1, 1, 1], - padding='SAME') - img_patches = tf.squeeze(img_patches) - one_patches = tf.squeeze(one_patches) - img_patches_resh = tf.reshape(img_patches, shape=(img_patches.shape[0] * img_patches.shape[1], - img_height_model, img_width_model, 3)) - pred_patches = model.predict(img_patches_resh, batch_size=n_batch_inference) - one_patches = tf.reshape(one_patches, shape=(img_patches.shape[0] * img_patches.shape[1], - img_height_model, img_width_model, 3)) - x = tf.range(img.shape[1]) - y = tf.range(img.shape[0]) - x, y = tf.meshgrid(x, y) - indices = tf.stack([y, x], axis=-1) - - indices_patches = tf.image.extract_patches( - images=tf.expand_dims(indices, axis=0), - sizes=[1, img_height_model, img_width_model, 1], - strides=[1, stride_y, stride_x, 1], - rates=[1, 1, 1, 1], - padding='SAME') - indices_patches = tf.squeeze(indices_patches) - indices_patches = tf.reshape(indices_patches, shape=(img_patches.shape[0] * img_patches.shape[1], - img_height_model, img_width_model, 2)) - margin_y = int( 0.5 * (img_height_model - stride_y) ) - margin_x = int( 0.5 * (img_width_model - stride_x) ) - - mask_margin = np.zeros((img_height_model, img_width_model)) - mask_margin[margin_y:img_height_model - margin_y, - margin_x:img_width_model - margin_x] = 1 - - indices_patches_array = indices_patches.numpy() - for i in range(indices_patches_array.shape[0]): - indices_patches_array[i,:,:,0] = indices_patches_array[i,:,:,0]*mask_margin - indices_patches_array[i,:,:,1] = indices_patches_array[i,:,:,1]*mask_margin - - reconstructed = tf.scatter_nd( - indices=indices_patches_array, - updates=pred_patches, - shape=(img.shape[0], img.shape[1], pred_patches.shape[-1])).numpy() - - prediction_true = np.argmax(reconstructed, axis=2).astype(np.uint8) - gc.collect() - return np.repeat(prediction_true[:, :, np.newaxis], 3, axis=2) - - def do_prediction_new_concept( - self, patches, img, model, - n_batch_inference=1, marginal_of_patch_percent=0.1, - thresholding_for_some_classes_in_light_version=False, - thresholding_for_artificial_class_in_light_version=False): - - self.logger.debug("enter do_prediction_new_concept") - img_height_model = model.layers[-1].output_shape[1] - img_width_model = model.layers[-1].output_shape[2] - - if not patches: - img_h_page = img.shape[0] - img_w_page = img.shape[1] - img = img / 255.0 - img = resize_image(img, img_height_model, img_width_model) - - label_p_pred = model.predict(img[np.newaxis], verbose=0) - seg = np.argmax(label_p_pred, axis=3)[0] - - if thresholding_for_artificial_class_in_light_version: - #seg_text = label_p_pred[0,:,:,1] - #seg_text[seg_text<0.2] =0 - #seg_text[seg_text>0] =1 - #seg[seg_text==1]=1 - - seg_art = label_p_pred[0,:,:,4] - seg_art[seg_art<0.2] =0 - seg_art[seg_art>0] =1 - seg[seg_art==1]=4 - - seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) - prediction_true = resize_image(seg_color, img_h_page, img_w_page).astype(np.uint8) - return prediction_true , resize_image(label_p_pred[0, :, :, 1] , img_h_page, img_w_page) - - if img.shape[0] < img_height_model: - img = resize_image(img, img_height_model, img.shape[1]) - if img.shape[1] < img_width_model: - img = resize_image(img, img.shape[0], img_width_model) - - self.logger.debug("Patch size: %sx%s", img_height_model, img_width_model) - margin = int(marginal_of_patch_percent * img_height_model) - width_mid = img_width_model - 2 * margin - height_mid = img_height_model - 2 * margin - img = img / 255.0 - img = img.astype(np.float16) - img_h = img.shape[0] - img_w = img.shape[1] - prediction_true = np.zeros((img_h, img_w, 3)) - confidence_matrix = np.zeros((img_h, img_w)) - mask_true = np.zeros((img_h, img_w)) - nxf = img_w / float(width_mid) - nyf = img_h / float(height_mid) - nxf = int(nxf) + 1 if nxf > int(nxf) else int(nxf) - nyf = int(nyf) + 1 if nyf > int(nyf) else int(nyf) - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - img_patch = np.zeros((n_batch_inference, img_height_model, img_width_model, 3)) - for i in range(nxf): - for j in range(nyf): - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - else: - index_x_d = i * width_mid - index_x_u = index_x_d + img_width_model - if j == 0: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - else: - index_y_d = j * height_mid - index_y_u = index_y_d + img_height_model - if index_x_u > img_w: - index_x_u = img_w - index_x_d = img_w - img_width_model - if index_y_u > img_h: - index_y_u = img_h - index_y_d = img_h - img_height_model - - list_i_s.append(i) - list_j_s.append(j) - list_x_u.append(index_x_u) - list_x_d.append(index_x_d) - list_y_d.append(index_y_d) - list_y_u.append(index_y_u) - - img_patch[batch_indexer] = img[index_y_d:index_y_u, index_x_d:index_x_u] - batch_indexer += 1 - - if (batch_indexer == n_batch_inference or - # last batch - i == nxf - 1 and j == nyf - 1): - self.logger.debug("predicting patches on %s", str(img_patch.shape)) - label_p_pred = model.predict(img_patch,verbose=0) - seg = np.argmax(label_p_pred, axis=3) - - if thresholding_for_some_classes_in_light_version: - seg_art = label_p_pred[:,:,:,4] - seg_art[seg_art<0.2] =0 - seg_art[seg_art>0] =1 - - seg_line = label_p_pred[:,:,:,3] - seg_line[seg_line>0.1] =1 - seg_line[seg_line<1] =0 - - seg[seg_art==1]=4 - seg[(seg_line==1) & (seg==0)]=3 - if thresholding_for_artificial_class_in_light_version: - seg_art = label_p_pred[:,:,:,2] - - seg_art[seg_art<0.2] = 0 - seg_art[seg_art>0] =1 - - seg[seg_art==1]=2 - - indexer_inside_batch = 0 - for i_batch, j_batch in zip(list_i_s, list_j_s): - seg_in = seg[indexer_inside_batch] - - index_y_u_in = list_y_u[indexer_inside_batch] - index_y_d_in = list_y_d[indexer_inside_batch] - - index_x_u_in = list_x_u[indexer_inside_batch] - index_x_d_in = list_x_d[indexer_inside_batch] - - if i_batch == 0 and j_batch == 0: - prediction_true[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + 0:index_x_u_in - margin] = \ - seg_in[0:-margin or None, - 0:-margin or None, - np.newaxis] - confidence_matrix[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + 0:index_x_u_in - margin] = \ - label_p_pred[0, 0:-margin or None, - 0:-margin or None, - 1] - elif i_batch == nxf - 1 and j_batch == nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + margin:index_x_u_in - 0] = \ - seg_in[margin:, - margin:, - np.newaxis] - confidence_matrix[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + margin:index_x_u_in - 0] = \ - label_p_pred[0, margin:, - margin:, - 1] - elif i_batch == 0 and j_batch == nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + 0:index_x_u_in - margin] = \ - seg_in[margin:, - 0:-margin or None, - np.newaxis] - confidence_matrix[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + 0:index_x_u_in - margin] = \ - label_p_pred[0, margin:, - 0:-margin or None, - 1] - elif i_batch == nxf - 1 and j_batch == 0: - prediction_true[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - 0] = \ - seg_in[0:-margin or None, - margin:, - np.newaxis] - confidence_matrix[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - 0] = \ - label_p_pred[0, 0:-margin or None, - margin:, - 1] - elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + 0:index_x_u_in - margin] = \ - seg_in[margin:-margin or None, - 0:-margin or None, - np.newaxis] - confidence_matrix[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + 0:index_x_u_in - margin] = \ - label_p_pred[0, margin:-margin or None, - 0:-margin or None, - 1] - elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - 0] = \ - seg_in[margin:-margin or None, - margin:, - np.newaxis] - confidence_matrix[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - 0] = \ - label_p_pred[0, margin:-margin or None, - margin:, - 1] - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: - prediction_true[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - margin] = \ - seg_in[0:-margin or None, - margin:-margin or None, - np.newaxis] - confidence_matrix[index_y_d_in + 0:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - margin] = \ - label_p_pred[0, 0:-margin or None, - margin:-margin or None, - 1] - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: - prediction_true[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + margin:index_x_u_in - margin] = \ - seg_in[margin:, - margin:-margin or None, - np.newaxis] - confidence_matrix[index_y_d_in + margin:index_y_u_in - 0, - index_x_d_in + margin:index_x_u_in - margin] = \ - label_p_pred[0, margin:, - margin:-margin or None, - 1] - else: - prediction_true[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - margin] = \ - seg_in[margin:-margin or None, - margin:-margin or None, - np.newaxis] - confidence_matrix[index_y_d_in + margin:index_y_u_in - margin, - index_x_d_in + margin:index_x_u_in - margin] = \ - label_p_pred[0, margin:-margin or None, - margin:-margin or None, - 1] - indexer_inside_batch += 1 - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - img_patch[:] = 0 - - prediction_true = prediction_true.astype(np.uint8) - gc.collect() - return prediction_true, confidence_matrix - - def extract_page(self): - self.logger.debug("enter extract_page") - cont_page = [] - if not self.ignore_page_extraction: - img = cv2.GaussianBlur(self.image, (5, 5), 0) - img_page_prediction = self.do_prediction(False, img, self.model_page) - imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - thresh = cv2.dilate(thresh, KERNEL, iterations=3) - contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - if len(contours)>0: - cnt_size = np.array([cv2.contourArea(contours[j]) - for j in range(len(contours))]) - cnt = contours[np.argmax(cnt_size)] - x, y, w, h = cv2.boundingRect(cnt) - if x <= 30: - w += x - x = 0 - if (self.image.shape[1] - (x + w)) <= 30: - w = w + (self.image.shape[1] - (x + w)) - if y <= 30: - h = h + y - y = 0 - if (self.image.shape[0] - (y + h)) <= 30: - h = h + (self.image.shape[0] - (y + h)) - box = [x, y, w, h] - else: - box = [0, 0, img.shape[1], img.shape[0]] - cropped_page, page_coord = crop_image_inside_box(box, self.image) - cont_page.append(np.array([[page_coord[2], page_coord[0]], - [page_coord[3], page_coord[0]], - [page_coord[3], page_coord[1]], - [page_coord[2], page_coord[1]]])) - self.logger.debug("exit extract_page") - else: - box = [0, 0, self.image.shape[1], self.image.shape[0]] - cropped_page, page_coord = crop_image_inside_box(box, self.image) - cont_page.append(np.array([[page_coord[2], page_coord[0]], - [page_coord[3], page_coord[0]], - [page_coord[3], page_coord[1]], - [page_coord[2], page_coord[1]]])) - return cropped_page, page_coord, cont_page - - def early_page_for_num_of_column_classification(self,img_bin): - if not self.ignore_page_extraction: - self.logger.debug("enter early_page_for_num_of_column_classification") - if self.input_binary: - img = np.copy(img_bin).astype(np.uint8) - else: - img = self.imread() - img = cv2.GaussianBlur(img, (5, 5), 0) - img_page_prediction = self.do_prediction(False, img, self.model_page) - - imgray = cv2.cvtColor(img_page_prediction, cv2.COLOR_BGR2GRAY) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - thresh = cv2.dilate(thresh, KERNEL, iterations=3) - contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - if len(contours)>0: - cnt_size = np.array([cv2.contourArea(contours[j]) - for j in range(len(contours))]) - cnt = contours[np.argmax(cnt_size)] - box = cv2.boundingRect(cnt) - else: - box = [0, 0, img.shape[1], img.shape[0]] - cropped_page, page_coord = crop_image_inside_box(box, img) - - self.logger.debug("exit early_page_for_num_of_column_classification") - else: - img = self.imread() - box = [0, 0, img.shape[1], img.shape[0]] - cropped_page, page_coord = crop_image_inside_box(box, img) - return cropped_page, page_coord - - def extract_text_regions_new(self, img, patches, cols): - self.logger.debug("enter extract_text_regions") - img_height_h = img.shape[0] - img_width_h = img.shape[1] - model_region = self.model_region_fl if patches else self.model_region_fl_np - - if self.light_version: - pass - elif not patches: - img = otsu_copy_binary(img).astype(np.uint8) - prediction_regions = None - elif cols: - img = otsu_copy_binary(img).astype(np.uint8) - if cols == 1: - img = resize_image(img, int(img_height_h * 1000 / float(img_width_h)), 1000).astype(np.uint8) - elif cols == 2: - img = resize_image(img, int(img_height_h * 1300 / float(img_width_h)), 1300).astype(np.uint8) - elif cols == 3: - img = resize_image(img, int(img_height_h * 1600 / float(img_width_h)), 1600).astype(np.uint8) - elif cols == 4: - img = resize_image(img, int(img_height_h * 1900 / float(img_width_h)), 1900).astype(np.uint8) - elif cols == 5: - img = resize_image(img, int(img_height_h * 2200 / float(img_width_h)), 2200).astype(np.uint8) - else: - img = resize_image(img, int(img_height_h * 2500 / float(img_width_h)), 2500).astype(np.uint8) - - prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1, n_batch_inference=3) - prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) - self.logger.debug("exit extract_text_regions") - return prediction_regions, prediction_regions - - def extract_text_regions(self, img, patches, cols): - self.logger.debug("enter extract_text_regions") - img_height_h = img.shape[0] - img_width_h = img.shape[1] - model_region = self.model_region_fl if patches else self.model_region_fl_np - - if not patches: - img = otsu_copy_binary(img) - img = img.astype(np.uint8) - prediction_regions2 = None - elif cols: - if cols == 1: - img_height_new = int(img_height_h * 0.7) - img_width_new = int(img_width_h * 0.7) - elif cols == 2: - img_height_new = int(img_height_h * 0.4) - img_width_new = int(img_width_h * 0.4) - else: - img_height_new = int(img_height_h * 0.3) - img_width_new = int(img_width_h * 0.3) - img2 = otsu_copy_binary(img) - img2 = img2.astype(np.uint8) - img2 = resize_image(img2, img_height_new, img_width_new) - prediction_regions2 = self.do_prediction(patches, img2, model_region, marginal_of_patch_percent=0.1) - prediction_regions2 = resize_image(prediction_regions2, img_height_h, img_width_h) - - img = otsu_copy_binary(img).astype(np.uint8) - if cols == 1: - img = resize_image(img, int(img_height_h * 0.5), int(img_width_h * 0.5)).astype(np.uint8) - elif cols == 2 and img_width_h >= 2000: - img = resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)).astype(np.uint8) - elif cols == 3 and ((self.scale_x == 1 and img_width_h > 3000) or - (self.scale_x != 1 and img_width_h > 2800)): - img = resize_image(img, 2800 * img_height_h // img_width_h, 2800).astype(np.uint8) - elif cols == 4 and ((self.scale_x == 1 and img_width_h > 4000) or - (self.scale_x != 1 and img_width_h > 3700)): - img = resize_image(img, 3700 * img_height_h // img_width_h, 3700).astype(np.uint8) - elif cols == 4: - img = resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)).astype(np.uint8) - elif cols == 5 and self.scale_x == 1 and img_width_h > 5000: - img = resize_image(img, int(img_height_h * 0.7), int(img_width_h * 0.7)).astype(np.uint8) - elif cols == 5: - img = resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)).astype(np.uint8) - elif img_width_h > 5600: - img = resize_image(img, 5600 * img_height_h // img_width_h, 5600).astype(np.uint8) - else: - img = resize_image(img, int(img_height_h * 0.9), int(img_width_h * 0.9)).astype(np.uint8) - - prediction_regions = self.do_prediction(patches, img, model_region, marginal_of_patch_percent=0.1) - prediction_regions = resize_image(prediction_regions, img_height_h, img_width_h) - self.logger.debug("exit extract_text_regions") - return prediction_regions, prediction_regions2 - - def get_slopes_and_deskew_new_light2(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): - - polygons_of_textlines = return_contours_of_interested_region(textline_mask_tot,1,0.00001) - M_main_tot = [cv2.moments(polygons_of_textlines[j]) - for j in range(len(polygons_of_textlines))] - cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - - args_textlines = np.array(range(len(polygons_of_textlines))) - all_found_textline_polygons = [] - slopes = [] - all_box_coord =[] - - for index, con_region_ind in enumerate(contours_par): - results = [cv2.pointPolygonTest(con_region_ind, (cx_main_tot[ind], cy_main_tot[ind]), False) - for ind in args_textlines ] - results = np.array(results) - indexes_in = args_textlines[results==1] - textlines_ins = [polygons_of_textlines[ind] for ind in indexes_in] - - all_found_textline_polygons.append(textlines_ins[::-1]) - slopes.append(slope_deskew) - - _, crop_coor = crop_image_inside_box(boxes[index],image_page_rotated) - all_box_coord.append(crop_coor) - - return all_found_textline_polygons, boxes, contours, contours_par, all_box_coord, np.array(range(len(contours_par))), slopes - - def get_slopes_and_deskew_new_light(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): - if not len(contours): - return [], [], [], [], [], [], [] - self.logger.debug("enter get_slopes_and_deskew_new_light") - results = self.executor.map(partial(do_work_of_slopes_new_light, - textline_mask_tot_ea=textline_mask_tot, - image_page_rotated=image_page_rotated, - slope_deskew=slope_deskew,textline_light=self.textline_light, - logger=self.logger,), - boxes, contours, contours_par, range(len(contours_par))) - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) - self.logger.debug("exit get_slopes_and_deskew_new_light") - return tuple(zip(*results)) - - def get_slopes_and_deskew_new(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, slope_deskew): - if not len(contours): - return [], [], [], [], [], [], [] - self.logger.debug("enter get_slopes_and_deskew_new") - results = self.executor.map(partial(do_work_of_slopes_new, - textline_mask_tot_ea=textline_mask_tot, - image_page_rotated=image_page_rotated, - slope_deskew=slope_deskew, - MAX_SLOPE=MAX_SLOPE, - KERNEL=KERNEL, - logger=self.logger, - plotter=self.plotter,), - boxes, contours, contours_par, range(len(contours_par))) - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) - self.logger.debug("exit get_slopes_and_deskew_new") - return tuple(zip(*results)) - - def get_slopes_and_deskew_new_curved(self, contours, contours_par, textline_mask_tot, image_page_rotated, boxes, mask_texts_only, num_col, scale_par, slope_deskew): - if not len(contours): - return [], [], [], [], [], [], [] - self.logger.debug("enter get_slopes_and_deskew_new_curved") - results = self.executor.map(partial(do_work_of_slopes_new_curved, - textline_mask_tot_ea=textline_mask_tot, - image_page_rotated=image_page_rotated, - mask_texts_only=mask_texts_only, - num_col=num_col, - scale_par=scale_par, - slope_deskew=slope_deskew, - MAX_SLOPE=MAX_SLOPE, - KERNEL=KERNEL, - logger=self.logger, - plotter=self.plotter,), - boxes, contours, contours_par, range(len(contours_par))) - #textline_polygons, boxes, text_regions, text_regions_par, box_coord, index_text_con, slopes = zip(*results) - self.logger.debug("exit get_slopes_and_deskew_new_curved") - return tuple(zip(*results)) - - def textline_contours(self, img, use_patches, scaler_h, scaler_w, num_col_classifier=None): - self.logger.debug('enter textline_contours') - - #img = img.astype(np.uint8) - img_org = np.copy(img) - img_h = img_org.shape[0] - img_w = img_org.shape[1] - img = resize_image(img_org, int(img_org.shape[0] * scaler_h), int(img_org.shape[1] * scaler_w)) - - prediction_textline = self.do_prediction( - use_patches, img, self.model_textline, - marginal_of_patch_percent=0.15, n_batch_inference=3, - thresholding_for_artificial_class_in_light_version=self.textline_light) - #if not self.textline_light: - #if num_col_classifier==1: - #prediction_textline_nopatch = self.do_prediction(False, img, self.model_textline) - #prediction_textline[:,:][prediction_textline_nopatch[:,:]==0] = 0 - - prediction_textline = resize_image(prediction_textline, img_h, img_w) - textline_mask_tot_ea_art = (prediction_textline[:,:]==2)*1 - - old_art = np.copy(textline_mask_tot_ea_art) - if not self.textline_light: - textline_mask_tot_ea_art = textline_mask_tot_ea_art.astype('uint8') - #textline_mask_tot_ea_art = cv2.dilate(textline_mask_tot_ea_art, KERNEL, iterations=1) - prediction_textline[:,:][textline_mask_tot_ea_art[:,:]==1]=2 - - textline_mask_tot_ea_lines = (prediction_textline[:,:]==1)*1 - textline_mask_tot_ea_lines = textline_mask_tot_ea_lines.astype('uint8') - if not self.textline_light: - textline_mask_tot_ea_lines = cv2.dilate(textline_mask_tot_ea_lines, KERNEL, iterations=1) - - prediction_textline[:,:][textline_mask_tot_ea_lines[:,:]==1]=1 - if not self.textline_light: - prediction_textline[:,:][old_art[:,:]==1]=2 - - prediction_textline_longshot = self.do_prediction(False, img, self.model_textline) - prediction_textline_longshot_true_size = resize_image(prediction_textline_longshot, img_h, img_w) - - self.logger.debug('exit textline_contours') - return ((prediction_textline[:, :, 0]==1).astype(np.uint8), - (prediction_textline_longshot_true_size[:, :, 0]==1).astype(np.uint8)) - - - def do_work_of_slopes(self, q, poly, box_sub, boxes_per_process, textline_mask_tot, contours_per_process): - self.logger.debug('enter do_work_of_slopes') - slope_biggest = 0 - slopes_sub = [] - boxes_sub_new = [] - poly_sub = [] - for mv in range(len(boxes_per_process)): - crop_img, _ = crop_image_inside_box(boxes_per_process[mv], np.repeat(textline_mask_tot[:, :, np.newaxis], 3, axis=2)) - crop_img = crop_img[:, :, 0] - crop_img = cv2.erode(crop_img, KERNEL, iterations=2) - try: - textline_con, hierarchy = return_contours_of_image(crop_img) - textline_con_fil = filter_contours_area_of_image(crop_img, textline_con, hierarchy, max_area=1, min_area=0.0008) - y_diff_mean = find_contours_mean_y_diff(textline_con_fil) - sigma_des = max(1, int(y_diff_mean * (4.0 / 40.0))) - crop_img[crop_img > 0] = 1 - slope_corresponding_textregion = return_deskew_slop(crop_img, sigma_des, - map=self.executor.map, logger=self.logger, plotter=self.plotter) - except Exception as why: - self.logger.error(why) - slope_corresponding_textregion = MAX_SLOPE - - if slope_corresponding_textregion == MAX_SLOPE: - slope_corresponding_textregion = slope_biggest - slopes_sub.append(slope_corresponding_textregion) - - cnt_clean_rot = textline_contours_postprocessing( - crop_img, slope_corresponding_textregion, contours_per_process[mv], boxes_per_process[mv]) - - poly_sub.append(cnt_clean_rot) - boxes_sub_new.append(boxes_per_process[mv]) - - q.put(slopes_sub) - poly.put(poly_sub) - box_sub.put(boxes_sub_new) - self.logger.debug('exit do_work_of_slopes') - - def get_regions_light_v_extract_only_images(self,img,is_image_enhanced, num_col_classifier): - self.logger.debug("enter get_regions_extract_images_only") - erosion_hurts = False - img_org = np.copy(img) - img_height_h = img_org.shape[0] - img_width_h = img_org.shape[1] - - if num_col_classifier == 1: - img_w_new = 700 - elif num_col_classifier == 2: - img_w_new = 900 - elif num_col_classifier == 3: - img_w_new = 1500 - elif num_col_classifier == 4: - img_w_new = 1800 - elif num_col_classifier == 5: - img_w_new = 2200 - elif num_col_classifier == 6: - img_w_new = 2500 - img_h_new = int(img.shape[0] / float(img.shape[1]) * img_w_new) - img_resized = resize_image(img,img_h_new, img_w_new ) - - prediction_regions_org, _ = self.do_prediction_new_concept(True, img_resized, self.model_region) - - prediction_regions_org = resize_image(prediction_regions_org,img_height_h, img_width_h ) - image_page, page_coord, cont_page = self.extract_page() - - prediction_regions_org = prediction_regions_org[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - prediction_regions_org=prediction_regions_org[:,:,0] - - mask_lines_only = (prediction_regions_org[:,:] ==3)*1 - mask_texts_only = (prediction_regions_org[:,:] ==1)*1 - mask_images_only=(prediction_regions_org[:,:] ==2)*1 - - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - polygons_lines_xml = textline_con_fil = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) - - polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) - polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) - - text_regions_p_true = np.zeros(prediction_regions_org.shape) - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_lines, color=(3,3,3)) - - text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_texts, color=(1,1,1)) - - text_regions_p_true[text_regions_p_true.shape[0]-15:text_regions_p_true.shape[0], :] = 0 - text_regions_p_true[:, text_regions_p_true.shape[1]-15:text_regions_p_true.shape[1]] = 0 - - ##polygons_of_images = return_contours_of_interested_region(text_regions_p_true, 2, 0.0001) - polygons_of_images = return_contours_of_interested_region(text_regions_p_true, 2, 0.001) - image_boundary_of_doc = np.zeros((text_regions_p_true.shape[0], text_regions_p_true.shape[1])) - - ###image_boundary_of_doc[:6, :] = 1 - ###image_boundary_of_doc[text_regions_p_true.shape[0]-6:text_regions_p_true.shape[0], :] = 1 - - ###image_boundary_of_doc[:, :6] = 1 - ###image_boundary_of_doc[:, text_regions_p_true.shape[1]-6:text_regions_p_true.shape[1]] = 1 - - polygons_of_images_fin = [] - for ploy_img_ind in polygons_of_images: - """ - test_poly_image = np.zeros((text_regions_p_true.shape[0], text_regions_p_true.shape[1])) - test_poly_image = cv2.fillPoly(test_poly_image, pts=[ploy_img_ind], color=(1,1,1)) - - test_poly_image = test_poly_image + image_boundary_of_doc - test_poly_image_intersected_area = ( test_poly_image[:,:]==2 )*1 - - test_poly_image_intersected_area = test_poly_image_intersected_area.sum() - - if test_poly_image_intersected_area==0: - ##polygons_of_images_fin.append(ploy_img_ind) - - box = cv2.boundingRect(ploy_img_ind) - _, page_coord_img = crop_image_inside_box(box, text_regions_p_true) - # cont_page.append(np.array([[page_coord[2], page_coord[0]], - # [page_coord[3], page_coord[0]], - # [page_coord[3], page_coord[1]], - # [page_coord[2], page_coord[1]]])) - polygons_of_images_fin.append(np.array([[page_coord_img[2], page_coord_img[0]], - [page_coord_img[3], page_coord_img[0]], - [page_coord_img[3], page_coord_img[1]], - [page_coord_img[2], page_coord_img[1]]]) ) - """ - box = x, y, w, h = cv2.boundingRect(ploy_img_ind) - if h < 150 or w < 150: - pass - else: - _, page_coord_img = crop_image_inside_box(box, text_regions_p_true) - # cont_page.append(np.array([[page_coord[2], page_coord[0]], - # [page_coord[3], page_coord[0]], - # [page_coord[3], page_coord[1]], - # [page_coord[2], page_coord[1]]])) - polygons_of_images_fin.append(np.array([[page_coord_img[2], page_coord_img[0]], - [page_coord_img[3], page_coord_img[0]], - [page_coord_img[3], page_coord_img[1]], - [page_coord_img[2], page_coord_img[1]]])) - - self.logger.debug("exit get_regions_extract_images_only") - return text_regions_p_true, erosion_hurts, polygons_lines_xml, polygons_of_images_fin, image_page, page_coord, cont_page - - def get_regions_light_v(self,img,is_image_enhanced, num_col_classifier, skip_layout_and_reading_order=False): - self.logger.debug("enter get_regions_light_v") - t_in = time.time() - erosion_hurts = False - img_org = np.copy(img) - img_height_h = img_org.shape[0] - img_width_h = img_org.shape[1] - - #print(num_col_classifier,'num_col_classifier') - - if num_col_classifier == 1: - img_w_new = 1000 - elif num_col_classifier == 2: - img_w_new = 1500#1500 - elif num_col_classifier == 3: - img_w_new = 2000 - elif num_col_classifier == 4: - img_w_new = 2500 - elif num_col_classifier == 5: - img_w_new = 3000 - else: - img_w_new = 4000 - img_h_new = img_w_new * img_org.shape[0] // img_org.shape[1] - img_resized = resize_image(img,img_h_new, img_w_new ) - - t_bin = time.time() - #if (not self.input_binary) or self.full_layout: - #if self.input_binary: - #img_bin = np.copy(img_resized) - ###if (not self.input_binary and self.full_layout) or (not self.input_binary and num_col_classifier >= 30): - ###prediction_bin = self.do_prediction(True, img_resized, self.model_bin, n_batch_inference=5) - - ####print("inside bin ", time.time()-t_bin) - ###prediction_bin=prediction_bin[:,:,0] - ###prediction_bin = (prediction_bin[:,:]==0)*1 - ###prediction_bin = prediction_bin*255 - - ###prediction_bin =np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) - - ###prediction_bin = prediction_bin.astype(np.uint16) - ####img= np.copy(prediction_bin) - ###img_bin = np.copy(prediction_bin) - ###else: - ###img_bin = np.copy(img_resized) - if self.ocr and not self.input_binary: - prediction_bin = self.do_prediction(True, img_resized, self.model_bin, n_batch_inference=5) - prediction_bin = 255 * (prediction_bin[:,:,0] == 0) - prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) - prediction_bin = prediction_bin.astype(np.uint16) - #img= np.copy(prediction_bin) - img_bin = np.copy(prediction_bin) - else: - img_bin = np.copy(img_resized) - #print("inside 1 ", time.time()-t_in) - - ###textline_mask_tot_ea = self.run_textline(img_bin) - self.logger.debug("detecting textlines on %s with %d colors", str(img_resized.shape), len(np.unique(img_resized))) - textline_mask_tot_ea = self.run_textline(img_resized, num_col_classifier) - textline_mask_tot_ea = resize_image(textline_mask_tot_ea,img_height_h, img_width_h ) - - #print(self.image_org.shape) - #cv2.imwrite('out_13.png', self.image_page_org_size) - - #plt.imshwo(self.image_page_org_size) - #plt.show() - if not skip_layout_and_reading_order: - #print("inside 2 ", time.time()-t_in) - if num_col_classifier == 1 or num_col_classifier == 2: - if self.image_org.shape[0]/self.image_org.shape[1] > 2.5: - self.logger.debug("resized to %dx%d for %d cols", - img_resized.shape[1], img_resized.shape[0], num_col_classifier) - prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( - True, img_resized, self.model_region_1_2, n_batch_inference=1, - thresholding_for_some_classes_in_light_version=True) - else: - prediction_regions_org = np.zeros((self.image_org.shape[0], self.image_org.shape[1], 3)) - confidence_matrix = np.zeros((self.image_org.shape[0], self.image_org.shape[1])) - prediction_regions_page, confidence_matrix_page = self.do_prediction_new_concept( - False, self.image_page_org_size, self.model_region_1_2, n_batch_inference=1, - thresholding_for_artificial_class_in_light_version=True) - ys = slice(*self.page_coord[0:2]) - xs = slice(*self.page_coord[2:4]) - prediction_regions_org[ys, xs] = prediction_regions_page - confidence_matrix[ys, xs] = confidence_matrix_page - - else: - new_h = (900+ (num_col_classifier-3)*100) - img_resized = resize_image(img_bin, int(new_h * img_bin.shape[0] /img_bin.shape[1]), new_h) - self.logger.debug("resized to %dx%d (new_h=%d) for %d cols", - img_resized.shape[1], img_resized.shape[0], new_h, num_col_classifier) - prediction_regions_org, confidence_matrix = self.do_prediction_new_concept( - True, img_resized, self.model_region_1_2, n_batch_inference=2, - thresholding_for_some_classes_in_light_version=True) - ###prediction_regions_org = self.do_prediction(True, img_bin, self.model_region, n_batch_inference=3, thresholding_for_some_classes_in_light_version=True) - #print("inside 3 ", time.time()-t_in) - #plt.imshow(prediction_regions_org[:,:,0]) - #plt.show() - - prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - confidence_matrix = resize_image(confidence_matrix, img_height_h, img_width_h ) - img_bin = resize_image(img_bin, img_height_h, img_width_h ) - prediction_regions_org=prediction_regions_org[:,:,0] - - mask_lines_only = (prediction_regions_org[:,:] ==3)*1 - mask_texts_only = (prediction_regions_org[:,:] ==1)*1 - mask_texts_only = mask_texts_only.astype('uint8') - - ##if num_col_classifier == 1 or num_col_classifier == 2: - ###mask_texts_only = cv2.erode(mask_texts_only, KERNEL, iterations=1) - ##mask_texts_only = cv2.dilate(mask_texts_only, KERNEL, iterations=1) - - mask_texts_only = cv2.dilate(mask_texts_only, kernel=np.ones((2,2), np.uint8), iterations=1) - mask_images_only=(prediction_regions_org[:,:] ==2)*1 - - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - test_khat = np.zeros(prediction_regions_org.shape) - test_khat = cv2.fillPoly(test_khat, pts=polygons_lines_xml, color=(1,1,1)) - - #plt.imshow(test_khat[:,:]) - #plt.show() - #for jv in range(1): - #print(jv, hir_lines_xml[0][232][3]) - #test_khat = np.zeros(prediction_regions_org.shape) - #test_khat = cv2.fillPoly(test_khat, pts = [polygons_lines_xml[232]], color=(1,1,1)) - #plt.imshow(test_khat[:,:]) - #plt.show() - - polygons_lines_xml = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) - - test_khat = np.zeros(prediction_regions_org.shape) - test_khat = cv2.fillPoly(test_khat, pts = polygons_lines_xml, color=(1,1,1)) - - #plt.imshow(test_khat[:,:]) - #plt.show() - #sys.exit() - - polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) - ##polygons_of_only_texts = self.dilate_textregions_contours(polygons_of_only_texts) - polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) - - text_regions_p_true = np.zeros(prediction_regions_org.shape) - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts=polygons_of_only_lines, color=(3,3,3)) - - text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_texts, color=(1,1,1)) - - #plt.imshow(textline_mask_tot_ea) - #plt.show() - - textline_mask_tot_ea[(text_regions_p_true==0) | (text_regions_p_true==4) ] = 0 - - #plt.imshow(textline_mask_tot_ea) - #plt.show() - #print("inside 4 ", time.time()-t_in) - self.logger.debug("exit get_regions_light_v") - return text_regions_p_true, erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin, confidence_matrix - else: - img_bin = resize_image(img_bin,img_height_h, img_width_h ) - self.logger.debug("exit get_regions_light_v") - return None, erosion_hurts, None, textline_mask_tot_ea, img_bin, None - - def get_regions_from_xy_2models(self,img,is_image_enhanced, num_col_classifier): - self.logger.debug("enter get_regions_from_xy_2models") - erosion_hurts = False - img_org = np.copy(img) - img_height_h = img_org.shape[0] - img_width_h = img_org.shape[1] - - ratio_y=1.3 - ratio_x=1 - - img = resize_image(img_org, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - prediction_regions_org_y = self.do_prediction(True, img, self.model_region) - prediction_regions_org_y = resize_image(prediction_regions_org_y, img_height_h, img_width_h ) - - #plt.imshow(prediction_regions_org_y[:,:,0]) - #plt.show() - prediction_regions_org_y = prediction_regions_org_y[:,:,0] - mask_zeros_y = (prediction_regions_org_y[:,:]==0)*1 - - ##img_only_regions_with_sep = ( (prediction_regions_org_y[:,:] != 3) & (prediction_regions_org_y[:,:] != 0) )*1 - img_only_regions_with_sep = (prediction_regions_org_y == 1).astype(np.uint8) - try: - img_only_regions = cv2.erode(img_only_regions_with_sep[:,:], KERNEL, iterations=20) - _, _ = find_num_col(img_only_regions, num_col_classifier, self.tables, multiplier=6.0) - img = resize_image(img_org, int(img_org.shape[0]), int(img_org.shape[1]*(1.2 if is_image_enhanced else 1))) - - prediction_regions_org = self.do_prediction(True, img, self.model_region) - prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - - prediction_regions_org=prediction_regions_org[:,:,0] - prediction_regions_org[(prediction_regions_org[:,:]==1) & (mask_zeros_y[:,:]==1)]=0 - - img = resize_image(img_org, int(img_org.shape[0]), int(img_org.shape[1])) - - prediction_regions_org2 = self.do_prediction(True, img, self.model_region_p2, marginal_of_patch_percent=0.2) - prediction_regions_org2=resize_image(prediction_regions_org2, img_height_h, img_width_h ) - - mask_zeros2 = (prediction_regions_org2[:,:,0] == 0) - mask_lines2 = (prediction_regions_org2[:,:,0] == 3) - text_sume_early = (prediction_regions_org[:,:] == 1).sum() - prediction_regions_org_copy = np.copy(prediction_regions_org) - prediction_regions_org_copy[(prediction_regions_org_copy[:,:]==1) & (mask_zeros2[:,:]==1)] = 0 - text_sume_second = ((prediction_regions_org_copy[:,:]==1)*1).sum() - rate_two_models = 100. * text_sume_second / text_sume_early - - self.logger.info("ratio_of_two_models: %s", rate_two_models) - if not(is_image_enhanced and rate_two_models < RATIO_OF_TWO_MODEL_THRESHOLD): - prediction_regions_org = np.copy(prediction_regions_org_copy) - - prediction_regions_org[(mask_lines2[:,:]==1) & (prediction_regions_org[:,:]==0)]=3 - mask_lines_only=(prediction_regions_org[:,:]==3)*1 - prediction_regions_org = cv2.erode(prediction_regions_org[:,:], KERNEL, iterations=2) - prediction_regions_org = cv2.dilate(prediction_regions_org[:,:], KERNEL, iterations=2) - - if rate_two_models<=40: - if self.input_binary: - prediction_bin = np.copy(img_org) - else: - prediction_bin = self.do_prediction(True, img_org, self.model_bin, n_batch_inference=5) - prediction_bin = resize_image(prediction_bin, img_height_h, img_width_h ) - prediction_bin = 255 * (prediction_bin[:,:,0]==0) - prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) - - ratio_y=1 - ratio_x=1 - - img = resize_image(prediction_bin, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - - prediction_regions_org = self.do_prediction(True, img, self.model_region) - prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - prediction_regions_org=prediction_regions_org[:,:,0] - - mask_lines_only=(prediction_regions_org[:,:]==3)*1 - - mask_texts_only=(prediction_regions_org[:,:]==1)*1 - mask_images_only=(prediction_regions_org[:,:]==2)*1 - - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - polygons_lines_xml = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) - - polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only, 1, 0.00001) - polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only, 1, 0.00001) - - text_regions_p_true = np.zeros(prediction_regions_org.shape) - text_regions_p_true = cv2.fillPoly(text_regions_p_true,pts = polygons_of_only_lines, color=(3, 3, 3)) - text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 - - text_regions_p_true=cv2.fillPoly(text_regions_p_true,pts=polygons_of_only_texts, color=(1,1,1)) - - self.logger.debug("exit get_regions_from_xy_2models") - return text_regions_p_true, erosion_hurts, polygons_lines_xml - except: - if self.input_binary: - prediction_bin = np.copy(img_org) - prediction_bin = self.do_prediction(True, img_org, self.model_bin, n_batch_inference=5) - prediction_bin = resize_image(prediction_bin, img_height_h, img_width_h ) - prediction_bin = 255 * (prediction_bin[:,:,0]==0) - prediction_bin = np.repeat(prediction_bin[:, :, np.newaxis], 3, axis=2) - else: - prediction_bin = np.copy(img_org) - ratio_y=1 - ratio_x=1 - - - img = resize_image(prediction_bin, int(img_org.shape[0]*ratio_y), int(img_org.shape[1]*ratio_x)) - prediction_regions_org = self.do_prediction(True, img, self.model_region) - prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - prediction_regions_org=prediction_regions_org[:,:,0] - - #mask_lines_only=(prediction_regions_org[:,:]==3)*1 - #img = resize_image(img_org, int(img_org.shape[0]*1), int(img_org.shape[1]*1)) - - #prediction_regions_org = self.do_prediction(True, img, self.model_region) - - #prediction_regions_org = resize_image(prediction_regions_org, img_height_h, img_width_h ) - - #prediction_regions_org = prediction_regions_org[:,:,0] - - #prediction_regions_org[(prediction_regions_org[:,:] == 1) & (mask_zeros_y[:,:] == 1)]=0 - - - mask_lines_only = (prediction_regions_org == 3)*1 - mask_texts_only = (prediction_regions_org == 1)*1 - mask_images_only= (prediction_regions_org == 2)*1 - - polygons_lines_xml, hir_lines_xml = return_contours_of_image(mask_lines_only) - polygons_lines_xml = filter_contours_area_of_image( - mask_lines_only, polygons_lines_xml, hir_lines_xml, max_area=1, min_area=0.00001) - - polygons_of_only_texts = return_contours_of_interested_region(mask_texts_only,1,0.00001) - polygons_of_only_lines = return_contours_of_interested_region(mask_lines_only,1,0.00001) - - text_regions_p_true = np.zeros(prediction_regions_org.shape) - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_lines, color=(3,3,3)) - - text_regions_p_true[:,:][mask_images_only[:,:] == 1] = 2 - text_regions_p_true = cv2.fillPoly(text_regions_p_true, pts = polygons_of_only_texts, color=(1,1,1)) - - erosion_hurts = True - self.logger.debug("exit get_regions_from_xy_2models") - return text_regions_p_true, erosion_hurts, polygons_lines_xml - - def do_order_of_regions_full_layout( - self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): - - self.logger.debug("enter do_order_of_regions_full_layout") - boxes = np.array(boxes, dtype=int) # to be on the safe side - cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours( - contours_only_text_parent) - cx_text_only_h, cy_text_only_h, x_min_text_only_h, _, _, _, y_cor_x_min_main_h = find_new_features_of_contours( - contours_only_text_parent_h) - - try: - arg_text_con = [] - for ii in range(len(cx_text_only)): - check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (x_min_text_only[ii] + 80 >= boxes[jj][0] and - x_min_text_only[ii] + 80 < boxes[jj][1] and - y_cor_x_min_main[ii] >= boxes[jj][2] and - y_cor_x_min_main[ii] < boxes[jj][3]): - arg_text_con.append(jj) - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) - arg_text_con_h = [] - for ii in range(len(cx_text_only_h)): - check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (x_min_text_only_h[ii] + 80 >= boxes[jj][0] and - x_min_text_only_h[ii] + 80 < boxes[jj][1] and - y_cor_x_min_main_h[ii] >= boxes[jj][2] and - y_cor_x_min_main_h[ii] < boxes[jj][3]): - arg_text_con_h.append(jj) - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only_h[ii] - boxes[jj][1]) ** 2 + - (cy_text_only_h[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con_h.append(ind_min) - args_contours_h = np.array(range(len(arg_text_con_h))) - - order_by_con_head = np.zeros(len(arg_text_con_h)) - order_by_con_main = np.zeros(len(arg_text_con)) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij in range(len(boxes)): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] - con_inter_box = [] - con_inter_box_h = [] - - for box in args_contours_box: - con_inter_box.append(contours_only_text_parent[box]) - - for box in args_contours_box_h: - con_inter_box_h.append(contours_only_text_parent_h[box]) - - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] - indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] - - for zahler, _ in enumerate(args_contours_box): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point - - for zahler, _ in enumerate(args_contours_box_h): - arg_order_v = indexes_sorted_head[zahler] - order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point - - for jji in range(len(id_of_texts)): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = [] - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - for tj1 in range(len(contours_only_text_parent_h)): - order_of_texts_tot.append(int(order_by_con_head[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) - - except Exception as why: - self.logger.error(why) - arg_text_con = [] - for ii in range(len(cx_text_only)): - check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (cx_text_only[ii] >= boxes[jj][0] and - cx_text_only[ii] < boxes[jj][1] and - cy_text_only[ii] >= boxes[jj][2] and - cy_text_only[ii] < boxes[jj][3]): - # this is valid if the center of region identify in which box it is located - arg_text_con.append(jj) - check_if_textregion_located_in_a_box = True - break - - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) - order_by_con_main = np.zeros(len(arg_text_con)) - - ############################# head - - arg_text_con_h = [] - for ii in range(len(cx_text_only_h)): - check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (cx_text_only_h[ii] >= boxes[jj][0] and - cx_text_only_h[ii] < boxes[jj][1] and - cy_text_only_h[ii] >= boxes[jj][2] and - cy_text_only_h[ii] < boxes[jj][3]): - # this is valid if the center of region identify in which box it is located - arg_text_con_h.append(jj) - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only_h[ii] - boxes[jj][1]) ** 2 + - (cy_text_only_h[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con_h.append(ind_min) - args_contours_h = np.array(range(len(arg_text_con_h))) - order_by_con_head = np.zeros(len(arg_text_con_h)) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij, _ in enumerate(boxes): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - args_contours_box_h = args_contours_h[np.array(arg_text_con_h) == iij] - con_inter_box = [] - con_inter_box_h = [] - - for box in args_contours_box: - con_inter_box.append(contours_only_text_parent[box]) - - for box in args_contours_box_h: - con_inter_box_h.append(contours_only_text_parent_h[box]) - - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_sorted_head = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 2] - indexes_by_type_head = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 2] - - for zahler, _ in enumerate(args_contours_box): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point - - for zahler, _ in enumerate(args_contours_box_h): - arg_order_v = indexes_sorted_head[zahler] - order_by_con_head[args_contours_box_h[indexes_by_type_head[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point - - for jji, _ in enumerate(id_of_texts): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = [] - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - for tj1 in range(len(contours_only_text_parent_h)): - order_of_texts_tot.append(int(order_by_con_head[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) - - self.logger.debug("exit do_order_of_regions_full_layout") - return order_text_new, id_of_texts_tot - - def do_order_of_regions_no_full_layout( - self, contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot): - - self.logger.debug("enter do_order_of_regions_no_full_layout") - boxes = np.array(boxes, dtype=int) # to be on the safe side - cx_text_only, cy_text_only, x_min_text_only, _, _, _, y_cor_x_min_main = find_new_features_of_contours( - contours_only_text_parent) - - try: - arg_text_con = [] - for ii in range(len(cx_text_only)): - check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (x_min_text_only[ii] + 80 >= boxes[jj][0] and - x_min_text_only[ii] + 80 < boxes[jj][1] and - y_cor_x_min_main[ii] >= boxes[jj][2] and - y_cor_x_min_main[ii] < boxes[jj][3]): - arg_text_con.append(jj) - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) - order_by_con_main = np.zeros(len(arg_text_con)) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij in range(len(boxes)): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - con_inter_box = [] - con_inter_box_h = [] - for i in range(len(args_contours_box)): - con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) - - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] - - for zahler, _ in enumerate(args_contours_box): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point - - for jji, _ in enumerate(id_of_texts): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = [] - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) - - except Exception as why: - self.logger.error(why) - arg_text_con = [] - for ii in range(len(cx_text_only)): - check_if_textregion_located_in_a_box = False - for jj in range(len(boxes)): - if (cx_text_only[ii] >= boxes[jj][0] and - cx_text_only[ii] < boxes[jj][1] and - cy_text_only[ii] >= boxes[jj][2] and - cy_text_only[ii] < boxes[jj][3]): - # this is valid if the center of region identify in which box it is located - arg_text_con.append(jj) - check_if_textregion_located_in_a_box = True - break - if not check_if_textregion_located_in_a_box: - dists_tr_from_box = [math.sqrt((cx_text_only[ii] - boxes[jj][1]) ** 2 + - (cy_text_only[ii] - boxes[jj][2]) ** 2) - for jj in range(len(boxes))] - ind_min = np.argmin(dists_tr_from_box) - arg_text_con.append(ind_min) - args_contours = np.array(range(len(arg_text_con))) - order_by_con_main = np.zeros(len(arg_text_con)) - - ref_point = 0 - order_of_texts_tot = [] - id_of_texts_tot = [] - for iij in range(len(boxes)): - ys = slice(*boxes[iij][2:4]) - xs = slice(*boxes[iij][0:2]) - args_contours_box = args_contours[np.array(arg_text_con) == iij] - con_inter_box = [] - con_inter_box_h = [] - for i in range(len(args_contours_box)): - con_inter_box.append(contours_only_text_parent[args_contours_box[i]]) - - indexes_sorted, matrix_of_orders, kind_of_texts_sorted, index_by_kind_sorted = order_of_regions( - textline_mask_tot[ys, xs], con_inter_box, con_inter_box_h, boxes[iij][2]) - - order_of_texts, id_of_texts = order_and_id_of_texts( - con_inter_box, con_inter_box_h, - matrix_of_orders, indexes_sorted, index_by_kind_sorted, kind_of_texts_sorted, ref_point) - - indexes_sorted_main = np.array(indexes_sorted)[np.array(kind_of_texts_sorted) == 1] - indexes_by_type_main = np.array(index_by_kind_sorted)[np.array(kind_of_texts_sorted) == 1] - - for zahler, _ in enumerate(args_contours_box): - arg_order_v = indexes_sorted_main[zahler] - order_by_con_main[args_contours_box[indexes_by_type_main[zahler]]] = \ - np.where(indexes_sorted == arg_order_v)[0][0] + ref_point - - for jji, _ in enumerate(id_of_texts): - order_of_texts_tot.append(order_of_texts[jji] + ref_point) - id_of_texts_tot.append(id_of_texts[jji]) - ref_point += len(id_of_texts) - - order_of_texts_tot = [] - - for tj1 in range(len(contours_only_text_parent)): - order_of_texts_tot.append(int(order_by_con_main[tj1])) - - order_text_new = [] - for iii in range(len(order_of_texts_tot)): - order_text_new.append(np.where(np.array(order_of_texts_tot) == iii)[0][0]) - - self.logger.debug("exit do_order_of_regions_no_full_layout") - return order_text_new, id_of_texts_tot - - def check_iou_of_bounding_box_and_contour_for_tables( - self, layout, table_prediction_early, pixel_table, num_col_classifier): - - layout_org = np.copy(layout) - layout_org[:,:,0][layout_org[:,:,0]==pixel_table] = 0 - layout = (layout[:,:,0]==pixel_table)*1 - - layout =np.repeat(layout[:, :, np.newaxis], 3, axis=2) - layout = layout.astype(np.uint8) - imgray = cv2.cvtColor(layout, cv2.COLOR_BGR2GRAY ) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - cnt_size = np.array([cv2.contourArea(contours[j]) - for j in range(len(contours))]) - - contours_new = [] - for i in range(len(contours)): - x, y, w, h = cv2.boundingRect(contours[i]) - iou = cnt_size[i] /float(w*h) *100 - if iou<80: - layout_contour = np.zeros((layout_org.shape[0], layout_org.shape[1])) - layout_contour= cv2.fillPoly(layout_contour,pts=[contours[i]] ,color=(1,1,1)) - - layout_contour_sum = layout_contour.sum(axis=0) - layout_contour_sum_diff = np.diff(layout_contour_sum) - layout_contour_sum_diff= np.abs(layout_contour_sum_diff) - layout_contour_sum_diff_smoothed= gaussian_filter1d(layout_contour_sum_diff, 10) - - peaks, _ = find_peaks(layout_contour_sum_diff_smoothed, height=0) - peaks= peaks[layout_contour_sum_diff_smoothed[peaks]>4] - - for j in range(len(peaks)): - layout_contour[:,peaks[j]-3+1:peaks[j]+1+3] = 0 - - layout_contour=cv2.erode(layout_contour[:,:], KERNEL, iterations=5) - layout_contour=cv2.dilate(layout_contour[:,:], KERNEL, iterations=5) - - layout_contour =np.repeat(layout_contour[:, :, np.newaxis], 3, axis=2) - layout_contour = layout_contour.astype(np.uint8) - - imgray = cv2.cvtColor(layout_contour, cv2.COLOR_BGR2GRAY ) - _, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_sep, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - for ji in range(len(contours_sep) ): - contours_new.append(contours_sep[ji]) - if num_col_classifier>=2: - only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1])) - only_recent_contour_image= cv2.fillPoly(only_recent_contour_image, pts=[contours_sep[ji]], color=(1,1,1)) - table_pixels_masked_from_early_pre = only_recent_contour_image * table_prediction_early - iou_in = 100. * table_pixels_masked_from_early_pre.sum() / only_recent_contour_image.sum() - #print(iou_in,'iou_in_in1') - - if iou_in>30: - layout_org= cv2.fillPoly(layout_org, pts=[contours_sep[ji]], color=3 * (pixel_table,)) - else: - pass - else: - layout_org= cv2.fillPoly(layout_org, pts=[contours_sep[ji]], color=3 * (pixel_table,)) - else: - contours_new.append(contours[i]) - if num_col_classifier>=2: - only_recent_contour_image = np.zeros((layout.shape[0],layout.shape[1])) - only_recent_contour_image= cv2.fillPoly(only_recent_contour_image,pts=[contours[i]] ,color=(1,1,1)) - - table_pixels_masked_from_early_pre = only_recent_contour_image * table_prediction_early - iou_in = 100. * table_pixels_masked_from_early_pre.sum() / only_recent_contour_image.sum() - #print(iou_in,'iou_in') - if iou_in>30: - layout_org= cv2.fillPoly(layout_org, pts=[contours[i]], color=3 * (pixel_table,)) - else: - pass - else: - layout_org= cv2.fillPoly(layout_org, pts=[contours[i]], color=3 * (pixel_table,)) - - return layout_org, contours_new - - def delete_separator_around(self, spliter_y,peaks_neg,image_by_region, pixel_line, pixel_table): - # format of subboxes: box=[x1, x2 , y1, y2] - pix_del = 100 - if len(image_by_region.shape)==3: - for i in range(len(spliter_y)-1): - for j in range(1,len(peaks_neg[i])-1): - ys = slice(int(spliter_y[i]), - int(spliter_y[i+1])) - xs = slice(peaks_neg[i][j] - pix_del, - peaks_neg[i][j] + pix_del) - image_by_region[ys,xs,0][image_by_region[ys,xs,0]==pixel_line] = 0 - image_by_region[ys,xs,0][image_by_region[ys,xs,1]==pixel_line] = 0 - image_by_region[ys,xs,0][image_by_region[ys,xs,2]==pixel_line] = 0 - - image_by_region[ys,xs,0][image_by_region[ys,xs,0]==pixel_table] = 0 - image_by_region[ys,xs,0][image_by_region[ys,xs,1]==pixel_table] = 0 - image_by_region[ys,xs,0][image_by_region[ys,xs,2]==pixel_table] = 0 - else: - for i in range(len(spliter_y)-1): - for j in range(1,len(peaks_neg[i])-1): - ys = slice(int(spliter_y[i]), - int(spliter_y[i+1])) - xs = slice(peaks_neg[i][j] - pix_del, - peaks_neg[i][j] + pix_del) - image_by_region[ys,xs][image_by_region[ys,xs]==pixel_line] = 0 - image_by_region[ys,xs][image_by_region[ys,xs]==pixel_table] = 0 - return image_by_region - - def add_tables_heuristic_to_layout( - self, image_regions_eraly_p, boxes, - slope_mean_hor, spliter_y, peaks_neg_tot, image_revised, - num_col_classifier, min_area, pixel_line): - - pixel_table =10 - image_revised_1 = self.delete_separator_around(spliter_y, peaks_neg_tot, image_revised, pixel_line, pixel_table) - - try: - image_revised_1[:,:30][image_revised_1[:,:30]==pixel_line] = 0 - image_revised_1[:,-30:][image_revised_1[:,-30:]==pixel_line] = 0 - except: - pass - boxes = np.array(boxes, dtype=int) # to be on the safe side - - img_comm_e = np.zeros(image_revised_1.shape) - img_comm = np.repeat(img_comm_e[:, :, np.newaxis], 3, axis=2) - - for indiv in np.unique(image_revised_1): - image_col=(image_revised_1==indiv)*255 - img_comm_in=np.repeat(image_col[:, :, np.newaxis], 3, axis=2) - img_comm_in=img_comm_in.astype(np.uint8) - - imgray = cv2.cvtColor(img_comm_in, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours,hirarchy=cv2.findContours(thresh.copy(), cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - if indiv==pixel_table: - main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area = 1, min_area = 0.001) - else: - main_contours = filter_contours_area_of_image_tables(thresh, contours, hirarchy, max_area = 1, min_area = min_area) - - img_comm = cv2.fillPoly(img_comm, pts = main_contours, color = (indiv, indiv, indiv)) - img_comm = img_comm.astype(np.uint8) - - if not self.isNaN(slope_mean_hor): - image_revised_last = np.zeros((image_regions_eraly_p.shape[0], image_regions_eraly_p.shape[1],3)) - for i in range(len(boxes)): - box_ys = slice(*boxes[i][2:4]) - box_xs = slice(*boxes[i][0:2]) - image_box = img_comm[box_ys, box_xs] - try: - image_box_tabels_1=(image_box[:,:,0]==pixel_table)*1 - contours_tab,_=return_contours_of_image(image_box_tabels_1) - contours_tab=filter_contours_area_of_image_tables(image_box_tabels_1,contours_tab,_,1,0.003) - image_box_tabels_1=(image_box[:,:,0]==pixel_line)*1 - - image_box_tabels_and_m_text=( (image_box[:,:,0]==pixel_table) | (image_box[:,:,0]==1) )*1 - image_box_tabels_and_m_text=image_box_tabels_and_m_text.astype(np.uint8) - - image_box_tabels_1=image_box_tabels_1.astype(np.uint8) - image_box_tabels_1 = cv2.dilate(image_box_tabels_1,KERNEL,iterations = 5) - - contours_table_m_text,_=return_contours_of_image(image_box_tabels_and_m_text) - image_box_tabels=np.repeat(image_box_tabels_1[:, :, np.newaxis], 3, axis=2) - - image_box_tabels=image_box_tabels.astype(np.uint8) - imgray = cv2.cvtColor(image_box_tabels, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_line,hierachy=cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) - - y_min_main_line ,y_max_main_line=find_features_of_contours(contours_line) - y_min_main_tab ,y_max_main_tab=find_features_of_contours(contours_tab) - - cx_tab_m_text,cy_tab_m_text ,x_min_tab_m_text , x_max_tab_m_text, y_min_tab_m_text ,y_max_tab_m_text, _= find_new_features_of_contours(contours_table_m_text) - cx_tabl,cy_tabl ,x_min_tabl , x_max_tabl, y_min_tabl ,y_max_tabl,_= find_new_features_of_contours(contours_tab) - - if len(y_min_main_tab )>0: - y_down_tabs=[] - y_up_tabs=[] - - for i_t in range(len(y_min_main_tab )): - y_down_tab=[] - y_up_tab=[] - for i_l in range(len(y_min_main_line)): - if y_min_main_tab[i_t]>y_min_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l] and y_min_main_tab[i_t]>y_max_main_line[i_l] and y_max_main_tab[i_t]>y_min_main_line[i_l]: - pass - elif y_min_main_tab[i_t]0: - for ijv in range(len(y_min_tab_col1)): - image_revised_last[int(y_min_tab_col1[ijv]):int(y_max_tab_col1[ijv]),:,:]=pixel_table - return image_revised_last - - def do_order_of_regions(self, *args, **kwargs): - if self.full_layout: - return self.do_order_of_regions_full_layout(*args, **kwargs) - return self.do_order_of_regions_no_full_layout(*args, **kwargs) - - def get_tables_from_model(self, img, num_col_classifier): - img_org = np.copy(img) - img_height_h = img_org.shape[0] - img_width_h = img_org.shape[1] - patches = False - if self.light_version: - prediction_table, _ = self.do_prediction_new_concept(patches, img, self.model_table) - prediction_table = prediction_table.astype(np.int16) - return prediction_table[:,:,0] - else: - if num_col_classifier < 4 and num_col_classifier > 2: - prediction_table = self.do_prediction(patches, img, self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img[:,:,:], -1), self.model_table) - pre_updown = cv2.flip(pre_updown, -1) - - prediction_table[:,:,0][pre_updown[:,:,0]==1]=1 - prediction_table = prediction_table.astype(np.int16) - - elif num_col_classifier ==2: - height_ext = 0 # img.shape[0] // 4 - h_start = height_ext // 2 - width_ext = img.shape[1] // 8 - w_start = width_ext // 2 - - img_new = np.zeros((img.shape[0] + height_ext, - img.shape[1] + width_ext, - img.shape[2])).astype(float) - ys = slice(h_start, h_start + img.shape[0]) - xs = slice(w_start, w_start + img.shape[1]) - img_new[ys, xs] = img - - prediction_ext = self.do_prediction(patches, img_new, self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img_new[:,:,:], -1), self.model_table) - pre_updown = cv2.flip(pre_updown, -1) - - prediction_table = prediction_ext[ys, xs] - prediction_table_updown = pre_updown[ys, xs] - - prediction_table[:,:,0][prediction_table_updown[:,:,0]==1]=1 - prediction_table = prediction_table.astype(np.int16) - elif num_col_classifier ==1: - height_ext = 0 # img.shape[0] // 4 - h_start = height_ext // 2 - width_ext = img.shape[1] // 4 - w_start = width_ext // 2 - - img_new =np.zeros((img.shape[0] + height_ext, - img.shape[1] + width_ext, - img.shape[2])).astype(float) - ys = slice(h_start, h_start + img.shape[0]) - xs = slice(w_start, w_start + img.shape[1]) - img_new[ys, xs] = img - - prediction_ext = self.do_prediction(patches, img_new, self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img_new[:,:,:], -1), self.model_table) - pre_updown = cv2.flip(pre_updown, -1) - - prediction_table = prediction_ext[ys, xs] - prediction_table_updown = pre_updown[ys, xs] - - prediction_table[:,:,0][prediction_table_updown[:,:,0]==1]=1 - prediction_table = prediction_table.astype(np.int16) - else: - prediction_table = np.zeros(img.shape) - img_w_half = img.shape[1] // 2 - - pre1 = self.do_prediction(patches, img[:,0:img_w_half,:], self.model_table) - pre2 = self.do_prediction(patches, img[:,img_w_half:,:], self.model_table) - pre_full = self.do_prediction(patches, img[:,:,:], self.model_table) - pre_updown = self.do_prediction(patches, cv2.flip(img[:,:,:], -1), self.model_table) - pre_updown = cv2.flip(pre_updown, -1) - - prediction_table_full_erode = cv2.erode(pre_full[:,:,0], KERNEL, iterations=4) - prediction_table_full_erode = cv2.dilate(prediction_table_full_erode, KERNEL, iterations=4) - - prediction_table_full_updown_erode = cv2.erode(pre_updown[:,:,0], KERNEL, iterations=4) - prediction_table_full_updown_erode = cv2.dilate(prediction_table_full_updown_erode, KERNEL, iterations=4) - - prediction_table[:,0:img_w_half,:] = pre1[:,:,:] - prediction_table[:,img_w_half:,:] = pre2[:,:,:] - - prediction_table[:,:,0][prediction_table_full_erode[:,:]==1]=1 - prediction_table[:,:,0][prediction_table_full_updown_erode[:,:]==1]=1 - prediction_table = prediction_table.astype(np.int16) - - #prediction_table_erode = cv2.erode(prediction_table[:,:,0], self.kernel, iterations=6) - #prediction_table_erode = cv2.dilate(prediction_table_erode, self.kernel, iterations=6) - - prediction_table_erode = cv2.erode(prediction_table[:,:,0], KERNEL, iterations=20) - prediction_table_erode = cv2.dilate(prediction_table_erode, KERNEL, iterations=20) - return prediction_table_erode.astype(np.int16) - - def run_graphics_and_columns_light( - self, text_regions_p_1, textline_mask_tot_ea, - num_col_classifier, num_column_is_classified, erosion_hurts, img_bin_light): - - #print(text_regions_p_1.shape, 'text_regions_p_1 shape run graphics') - #print(erosion_hurts, 'erosion_hurts') - t_in_gr = time.time() - img_g = self.imread(grayscale=True, uint8=True) - - img_g3 = np.zeros((img_g.shape[0], img_g.shape[1], 3)) - img_g3 = img_g3.astype(np.uint8) - img_g3[:, :, 0] = img_g[:, :] - img_g3[:, :, 1] = img_g[:, :] - img_g3[:, :, 2] = img_g[:, :] - - image_page, page_coord, cont_page = self.extract_page() - #print("inside graphics 1 ", time.time() - t_in_gr) - if self.tables: - table_prediction = self.get_tables_from_model(image_page, num_col_classifier) - else: - table_prediction = np.zeros((image_page.shape[0], image_page.shape[1])).astype(np.int16) - - if self.plotter: - self.plotter.save_page_image(image_page) - - text_regions_p_1 = text_regions_p_1[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - textline_mask_tot_ea = textline_mask_tot_ea[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - img_bin_light = img_bin_light[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - mask_images = (text_regions_p_1[:, :] == 2) * 1 - mask_images = mask_images.astype(np.uint8) - mask_images = cv2.erode(mask_images[:, :], KERNEL, iterations=10) - mask_lines = (text_regions_p_1[:, :] == 3) * 1 - mask_lines = mask_lines.astype(np.uint8) - img_only_regions_with_sep = ((text_regions_p_1[:, :] != 3) & (text_regions_p_1[:, :] != 0)) * 1 - img_only_regions_with_sep = img_only_regions_with_sep.astype(np.uint8) - - #print("inside graphics 2 ", time.time() - t_in_gr) - if erosion_hurts: - img_only_regions = np.copy(img_only_regions_with_sep[:,:]) - else: - img_only_regions = cv2.erode(img_only_regions_with_sep[:,:], KERNEL, iterations=6) - - ##print(img_only_regions.shape,'img_only_regions') - ##plt.imshow(img_only_regions[:,:]) - ##plt.show() - ##num_col, _ = find_num_col(img_only_regions, num_col_classifier, self.tables, multiplier=6.0) - try: - num_col, _ = find_num_col(img_only_regions, num_col_classifier, self.tables, multiplier=6.0) - num_col = num_col + 1 - if not num_column_is_classified: - num_col_classifier = num_col + 1 - except Exception as why: - self.logger.error(why) - num_col = None - #print("inside graphics 3 ", time.time() - t_in_gr) - return (num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, - text_regions_p_1, cont_page, table_prediction, textline_mask_tot_ea, img_bin_light) - - def run_graphics_and_columns_without_layout(self, textline_mask_tot_ea, img_bin_light): - #print(text_regions_p_1.shape, 'text_regions_p_1 shape run graphics') - #print(erosion_hurts, 'erosion_hurts') - t_in_gr = time.time() - img_g = self.imread(grayscale=True, uint8=True) - - img_g3 = np.zeros((img_g.shape[0], img_g.shape[1], 3)) - img_g3 = img_g3.astype(np.uint8) - img_g3[:, :, 0] = img_g[:, :] - img_g3[:, :, 1] = img_g[:, :] - img_g3[:, :, 2] = img_g[:, :] - - image_page, page_coord, cont_page = self.extract_page() - #print("inside graphics 1 ", time.time() - t_in_gr) - - textline_mask_tot_ea = textline_mask_tot_ea[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - img_bin_light = img_bin_light[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - - return page_coord, image_page, textline_mask_tot_ea, img_bin_light, cont_page - - def run_graphics_and_columns( - self, text_regions_p_1, - num_col_classifier, num_column_is_classified, erosion_hurts): - - t_in_gr = time.time() - img_g = self.imread(grayscale=True, uint8=True) - - img_g3 = np.zeros((img_g.shape[0], img_g.shape[1], 3)) - img_g3 = img_g3.astype(np.uint8) - img_g3[:, :, 0] = img_g[:, :] - img_g3[:, :, 1] = img_g[:, :] - img_g3[:, :, 2] = img_g[:, :] - - image_page, page_coord, cont_page = self.extract_page() - - if self.tables: - table_prediction = self.get_tables_from_model(image_page, num_col_classifier) - else: - table_prediction = np.zeros((image_page.shape[0], image_page.shape[1])).astype(np.int16) - - if self.plotter: - self.plotter.save_page_image(image_page) - - text_regions_p_1 = text_regions_p_1[page_coord[0] : page_coord[1], page_coord[2] : page_coord[3]] - mask_images = (text_regions_p_1[:, :] == 2) * 1 - mask_images = mask_images.astype(np.uint8) - mask_images = cv2.erode(mask_images[:, :], KERNEL, iterations=10) - mask_lines = (text_regions_p_1[:, :] == 3) * 1 - mask_lines = mask_lines.astype(np.uint8) - img_only_regions_with_sep = ((text_regions_p_1[:, :] != 3) & (text_regions_p_1[:, :] != 0)) * 1 - img_only_regions_with_sep = img_only_regions_with_sep.astype(np.uint8) - - if erosion_hurts: - img_only_regions = np.copy(img_only_regions_with_sep[:,:]) - else: - img_only_regions = cv2.erode(img_only_regions_with_sep[:,:], KERNEL, iterations=6) - try: - num_col, _ = find_num_col(img_only_regions, num_col_classifier, self.tables, multiplier=6.0) - num_col = num_col + 1 - if not num_column_is_classified: - num_col_classifier = num_col + 1 - except Exception as why: - self.logger.error(why) - num_col = None - return (num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, - text_regions_p_1, cont_page, table_prediction) - - def run_enhancement(self, light_version): - t_in = time.time() - self.logger.info("Resizing and enhancing image...") - is_image_enhanced, img_org, img_res, num_col_classifier, num_column_is_classified, img_bin = \ - self.resize_and_enhance_image_with_column_classifier(light_version) - self.logger.info("Image was %senhanced.", '' if is_image_enhanced else 'not ') - scale = 1 - if is_image_enhanced: - if self.allow_enhancement: - #img_res = img_res.astype(np.uint8) - self.get_image_and_scales(img_org, img_res, scale) - if self.plotter: - self.plotter.save_enhanced_image(img_res) - else: - self.get_image_and_scales_after_enhancing(img_org, img_res) - else: - if self.allow_enhancement: - self.get_image_and_scales(img_org, img_res, scale) - else: - self.get_image_and_scales(img_org, img_res, scale) - if self.allow_scaling: - img_org, img_res, is_image_enhanced = self.resize_image_with_column_classifier(is_image_enhanced, img_bin) - self.get_image_and_scales_after_enhancing(img_org, img_res) - #print("enhancement in ", time.time()-t_in) - return img_res, is_image_enhanced, num_col_classifier, num_column_is_classified - - def run_textline(self, image_page, num_col_classifier=None): - scaler_h_textline = 1#1.3 # 1.2#1.2 - scaler_w_textline = 1#1.3 # 0.9#1 - #print(image_page.shape) - textline_mask_tot_ea, _ = self.textline_contours(image_page, True, scaler_h_textline, scaler_w_textline, num_col_classifier) - if self.textline_light: - textline_mask_tot_ea = textline_mask_tot_ea.astype(np.int16) - - if self.plotter: - self.plotter.save_plot_of_textlines(textline_mask_tot_ea, image_page) - return textline_mask_tot_ea - - def run_deskew(self, textline_mask_tot_ea): - #print(textline_mask_tot_ea.shape, 'textline_mask_tot_ea deskew') - slope_deskew = return_deskew_slop(cv2.erode(textline_mask_tot_ea, KERNEL, iterations=2), 2, 30, True, - map=self.executor.map, logger=self.logger, plotter=self.plotter) - slope_first = 0 - - if self.plotter: - self.plotter.save_deskewed_image(slope_deskew) - self.logger.info("slope_deskew: %.2f°", slope_deskew) - return slope_deskew, slope_first - - def run_marginals( - self, image_page, textline_mask_tot_ea, mask_images, mask_lines, - num_col_classifier, slope_deskew, text_regions_p_1, table_prediction): - - image_page_rotated, textline_mask_tot = image_page[:, :], textline_mask_tot_ea[:, :] - textline_mask_tot[mask_images[:, :] == 1] = 0 - - text_regions_p_1[mask_lines[:, :] == 1] = 3 - text_regions_p = text_regions_p_1[:, :] - text_regions_p = np.array(text_regions_p) - - if num_col_classifier in (1, 2): - try: - regions_without_separators = (text_regions_p[:, :] == 1) * 1 - if self.tables: - regions_without_separators[table_prediction==1] = 1 - regions_without_separators = regions_without_separators.astype(np.uint8) - text_regions_p = get_marginals( - rotate_image(regions_without_separators, slope_deskew), text_regions_p, - num_col_classifier, slope_deskew, light_version=self.light_version, kernel=KERNEL) - except Exception as e: - self.logger.error("exception %s", e) - - if self.plotter: - self.plotter.save_plot_of_layout_main_all(text_regions_p, image_page) - self.plotter.save_plot_of_layout_main(text_regions_p, image_page) - return textline_mask_tot, text_regions_p, image_page_rotated - - def run_boxes_no_full_layout( - self, image_page, textline_mask_tot, text_regions_p, - slope_deskew, num_col_classifier, table_prediction, erosion_hurts): - - self.logger.debug('enter run_boxes_no_full_layout') - t_0_box = time.time() - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - _, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = rotation_not_90_func( - image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) - text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1]) - textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1]) - table_prediction_n = resize_image(table_prediction_n, text_regions_p.shape[0], text_regions_p.shape[1]) - regions_without_separators_d = (text_regions_p_1_n[:, :] == 1) * 1 - if self.tables: - regions_without_separators_d[table_prediction_n[:,:] == 1] = 1 - regions_without_separators = (text_regions_p[:, :] == 1) * 1 # ( (text_regions_p[:,:]==1) | (text_regions_p[:,:]==2) )*1 #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions) - #print(time.time()-t_0_box,'time box in 1') - if self.tables: - regions_without_separators[table_prediction ==1 ] = 1 - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - text_regions_p_1_n = None - textline_mask_tot_d = None - regions_without_separators_d = None - pixel_lines = 3 - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - _, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) - - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) - #print(time.time()-t_0_box,'time box in 2') - self.logger.info("num_col_classifier: %s", num_col_classifier) - - if num_col_classifier >= 3: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - regions_without_separators = regions_without_separators.astype(np.uint8) - regions_without_separators = cv2.erode(regions_without_separators[:, :], KERNEL, iterations=6) - else: - regions_without_separators_d = regions_without_separators_d.astype(np.uint8) - regions_without_separators_d = cv2.erode(regions_without_separators_d[:, :], KERNEL, iterations=6) - #print(time.time()-t_0_box,'time box in 3') - t1 = time.time() - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new( - splitter_y_new, regions_without_separators, matrix_of_lines_ch, - num_col_classifier, erosion_hurts, self.tables, self.right2left) - boxes_d = None - self.logger.debug("len(boxes): %s", len(boxes)) - #print(time.time()-t_0_box,'time box in 3.1') - - if self.tables: - if self.light_version: - pass - else: - text_regions_p_tables = np.copy(text_regions_p) - text_regions_p_tables[:,:][(table_prediction[:,:] == 1)] = 10 - pixel_line = 3 - img_revised_tab2 = self.add_tables_heuristic_to_layout( - text_regions_p_tables, boxes, 0, splitter_y_new, peaks_neg_tot_tables, text_regions_p_tables, - num_col_classifier , 0.000005, pixel_line) - #print(time.time()-t_0_box,'time box in 3.2') - img_revised_tab2, contoures_tables = self.check_iou_of_bounding_box_and_contour_for_tables( - img_revised_tab2, table_prediction, 10, num_col_classifier) - #print(time.time()-t_0_box,'time box in 3.3') - else: - boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new( - splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, - num_col_classifier, erosion_hurts, self.tables, self.right2left) - boxes = None - self.logger.debug("len(boxes): %s", len(boxes_d)) - - if self.tables: - if self.light_version: - pass - else: - text_regions_p_tables = np.copy(text_regions_p_1_n) - text_regions_p_tables =np.round(text_regions_p_tables) - text_regions_p_tables[:,:][(text_regions_p_tables[:,:] != 3) & (table_prediction_n[:,:] == 1)] = 10 - - pixel_line = 3 - img_revised_tab2 = self.add_tables_heuristic_to_layout( - text_regions_p_tables, boxes_d, 0, splitter_y_new_d, peaks_neg_tot_tables_d, text_regions_p_tables, - num_col_classifier, 0.000005, pixel_line) - img_revised_tab2_d,_ = self.check_iou_of_bounding_box_and_contour_for_tables( - img_revised_tab2, table_prediction_n, 10, num_col_classifier) - - img_revised_tab2_d_rotated = rotate_image(img_revised_tab2_d, -slope_deskew) - img_revised_tab2_d_rotated = np.round(img_revised_tab2_d_rotated) - img_revised_tab2_d_rotated = img_revised_tab2_d_rotated.astype(np.int8) - img_revised_tab2_d_rotated = resize_image(img_revised_tab2_d_rotated, text_regions_p.shape[0], text_regions_p.shape[1]) - #print(time.time()-t_0_box,'time box in 4') - self.logger.info("detecting boxes took %.1fs", time.time() - t1) - - if self.tables: - if self.light_version: - text_regions_p[:,:][table_prediction[:,:]==1] = 10 - img_revised_tab=text_regions_p[:,:] - else: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - img_revised_tab = np.copy(img_revised_tab2[:,:,0]) - img_revised_tab[:,:][(text_regions_p[:,:] == 1) & (img_revised_tab[:,:] != 10)] = 1 - else: - img_revised_tab = np.copy(text_regions_p[:,:]) - img_revised_tab[:,:][img_revised_tab[:,:] == 10] = 0 - img_revised_tab[:,:][img_revised_tab2_d_rotated[:,:,0] == 10] = 10 - - text_regions_p[:,:][text_regions_p[:,:]==10] = 0 - text_regions_p[:,:][img_revised_tab[:,:]==10] = 10 - else: - img_revised_tab=text_regions_p[:,:] - #img_revised_tab = text_regions_p[:, :] - if self.light_version: - polygons_of_images = return_contours_of_interested_region(text_regions_p, 2) - else: - polygons_of_images = return_contours_of_interested_region(img_revised_tab, 2) - - pixel_img = 4 - min_area_mar = 0.00001 - if self.light_version: - marginal_mask = (text_regions_p[:,:]==pixel_img)*1 - marginal_mask = marginal_mask.astype('uint8') - marginal_mask = cv2.dilate(marginal_mask, KERNEL, iterations=2) - - polygons_of_marginals = return_contours_of_interested_region(marginal_mask, 1, min_area_mar) - else: - polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar) - - pixel_img = 10 - contours_tables = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar) - #print(time.time()-t_0_box,'time box in 5') - self.logger.debug('exit run_boxes_no_full_layout') - return (polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, - regions_without_separators_d, boxes, boxes_d, - polygons_of_marginals, contours_tables) - - def run_boxes_full_layout( - self, image_page, textline_mask_tot, text_regions_p, - slope_deskew, num_col_classifier, img_only_regions, - table_prediction, erosion_hurts, img_bin_light): - - self.logger.debug('enter run_boxes_full_layout') - t_full0 = time.time() - if self.tables: - if self.light_version: - text_regions_p[:,:][table_prediction[:,:]==1] = 10 - img_revised_tab = text_regions_p[:,:] - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - image_page_rotated_n, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ - rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) - - text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) - textline_mask_tot_d = resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1]) - table_prediction_n = resize_image(table_prediction_n,text_regions_p.shape[0],text_regions_p.shape[1]) - - regions_without_separators_d = (text_regions_p_1_n[:,:] == 1)*1 - regions_without_separators_d[table_prediction_n[:,:] == 1] = 1 - else: - text_regions_p_1_n = None - textline_mask_tot_d = None - regions_without_separators_d = None - # regions_without_separators = ( text_regions_p[:,:]==1 | text_regions_p[:,:]==2 )*1 - #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions) - regions_without_separators = (text_regions_p[:,:] == 1)*1 - regions_without_separators[table_prediction == 1] = 1 - - else: - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - image_page_rotated_n, textline_mask_tot_d, text_regions_p_1_n, table_prediction_n = \ - rotation_not_90_func(image_page, textline_mask_tot, text_regions_p, table_prediction, slope_deskew) - - text_regions_p_1_n = resize_image(text_regions_p_1_n,text_regions_p.shape[0],text_regions_p.shape[1]) - textline_mask_tot_d = resize_image(textline_mask_tot_d,text_regions_p.shape[0],text_regions_p.shape[1]) - table_prediction_n = resize_image(table_prediction_n,text_regions_p.shape[0],text_regions_p.shape[1]) - - regions_without_separators_d = (text_regions_p_1_n[:,:] == 1)*1 - regions_without_separators_d[table_prediction_n[:,:] == 1] = 1 - else: - text_regions_p_1_n = None - textline_mask_tot_d = None - regions_without_separators_d = None - - # regions_without_separators = ( text_regions_p[:,:]==1 | text_regions_p[:,:]==2 )*1 - #self.return_regions_without_separators_new(text_regions_p[:,:,0],img_only_regions) - regions_without_separators = (text_regions_p[:,:] == 1)*1 - regions_without_separators[table_prediction == 1] = 1 - - pixel_lines=3 - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) - - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - num_col_d, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_lines) - - if num_col_classifier>=3: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - regions_without_separators = regions_without_separators.astype(np.uint8) - regions_without_separators = cv2.erode(regions_without_separators[:,:], KERNEL, iterations=6) - - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - regions_without_separators_d = regions_without_separators_d.astype(np.uint8) - regions_without_separators_d = cv2.erode(regions_without_separators_d[:,:], KERNEL, iterations=6) - else: - pass - - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new( - splitter_y_new, regions_without_separators, matrix_of_lines_ch, - num_col_classifier, erosion_hurts, self.tables, self.right2left) - text_regions_p_tables = np.copy(text_regions_p) - text_regions_p_tables[:,:][(table_prediction[:,:]==1)] = 10 - pixel_line = 3 - img_revised_tab2 = self.add_tables_heuristic_to_layout( - text_regions_p_tables, boxes, 0, splitter_y_new, peaks_neg_tot_tables, text_regions_p_tables, - num_col_classifier , 0.000005, pixel_line) - - img_revised_tab2,contoures_tables = self.check_iou_of_bounding_box_and_contour_for_tables( - img_revised_tab2, table_prediction, 10, num_col_classifier) - else: - boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new( - splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, - num_col_classifier, erosion_hurts, self.tables, self.right2left) - text_regions_p_tables = np.copy(text_regions_p_1_n) - text_regions_p_tables = np.round(text_regions_p_tables) - text_regions_p_tables[:,:][(text_regions_p_tables[:,:]!=3) & (table_prediction_n[:,:]==1)] = 10 - - pixel_line = 3 - img_revised_tab2 = self.add_tables_heuristic_to_layout( - text_regions_p_tables, boxes_d, 0, splitter_y_new_d, peaks_neg_tot_tables_d, text_regions_p_tables, - num_col_classifier, 0.000005, pixel_line) - - img_revised_tab2_d,_ = self.check_iou_of_bounding_box_and_contour_for_tables( - img_revised_tab2, table_prediction_n, 10, num_col_classifier) - img_revised_tab2_d_rotated = rotate_image(img_revised_tab2_d, -slope_deskew) - - img_revised_tab2_d_rotated = np.round(img_revised_tab2_d_rotated) - img_revised_tab2_d_rotated = img_revised_tab2_d_rotated.astype(np.int8) - - img_revised_tab2_d_rotated = resize_image(img_revised_tab2_d_rotated, text_regions_p.shape[0], text_regions_p.shape[1]) - - if np.abs(slope_deskew) < 0.13: - img_revised_tab = np.copy(img_revised_tab2[:,:,0]) - else: - img_revised_tab = np.copy(text_regions_p[:,:]) - img_revised_tab[:,:][img_revised_tab[:,:] == 10] = 0 - img_revised_tab[:,:][img_revised_tab2_d_rotated[:,:,0] == 10] = 10 - - ##img_revised_tab=img_revised_tab2[:,:,0] - #img_revised_tab=text_regions_p[:,:] - text_regions_p[:,:][text_regions_p[:,:]==10] = 0 - text_regions_p[:,:][img_revised_tab[:,:]==10] = 10 - #img_revised_tab[img_revised_tab2[:,:,0]==10] =10 - - pixel_img = 4 - min_area_mar = 0.00001 - - if self.light_version: - marginal_mask = (text_regions_p[:,:]==pixel_img)*1 - marginal_mask = marginal_mask.astype('uint8') - marginal_mask = cv2.dilate(marginal_mask, KERNEL, iterations=2) - - polygons_of_marginals = return_contours_of_interested_region(marginal_mask, 1, min_area_mar) - else: - polygons_of_marginals = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar) - - pixel_img = 10 - contours_tables = return_contours_of_interested_region(text_regions_p, pixel_img, min_area_mar) - - # set first model with second model - text_regions_p[:, :][text_regions_p[:, :] == 2] = 5 - text_regions_p[:, :][text_regions_p[:, :] == 3] = 6 - text_regions_p[:, :][text_regions_p[:, :] == 4] = 8 - - image_page = image_page.astype(np.uint8) - #print("full inside 1", time.time()- t_full0) - regions_fully, regions_fully_only_drop = self.extract_text_regions_new( - img_bin_light if self.light_version else image_page, - False, cols=num_col_classifier) - #print("full inside 2", time.time()- t_full0) - # 6 is the separators lable in old full layout model - # 4 is the drop capital class in old full layout model - # in the new full layout drop capital is 3 and separators are 5 - - text_regions_p[:,:][regions_fully[:,:,0]==5]=6 - ###regions_fully[:, :, 0][regions_fully_only_drop[:, :, 0] == 3] = 4 - - #text_regions_p[:,:][regions_fully[:,:,0]==6]=6 - ##regions_fully_only_drop = put_drop_out_from_only_drop_model(regions_fully_only_drop, text_regions_p) - ##regions_fully[:, :, 0][regions_fully_only_drop[:, :, 0] == 4] = 4 - drop_capital_label_in_full_layout_model = 3 - - drops = (regions_fully[:,:,0]==drop_capital_label_in_full_layout_model)*1 - drops= drops.astype(np.uint8) - - regions_fully[:,:,0][regions_fully[:,:,0]==drop_capital_label_in_full_layout_model] = 1 - - drops = cv2.erode(drops[:,:], KERNEL, iterations=1) - regions_fully[:,:,0][drops[:,:]==1] = drop_capital_label_in_full_layout_model - - regions_fully = putt_bb_of_drop_capitals_of_model_in_patches_in_layout( - regions_fully, drop_capital_label_in_full_layout_model, text_regions_p) - ##regions_fully_np, _ = self.extract_text_regions(image_page, False, cols=num_col_classifier) - ##if num_col_classifier > 2: - ##regions_fully_np[:, :, 0][regions_fully_np[:, :, 0] == 4] = 0 - ##else: - ##regions_fully_np = filter_small_drop_capitals_from_no_patch_layout(regions_fully_np, text_regions_p) - - ###regions_fully = boosting_headers_by_longshot_region_segmentation(regions_fully, regions_fully_np, img_only_regions) - # plt.imshow(regions_fully[:,:,0]) - # plt.show() - text_regions_p[:, :][regions_fully[:, :, 0] == drop_capital_label_in_full_layout_model] = 4 - ####text_regions_p[:, :][regions_fully_np[:, :, 0] == 4] = 4 - #plt.imshow(text_regions_p) - #plt.show() - ####if not self.tables: - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - _, textline_mask_tot_d, text_regions_p_1_n, regions_fully_n = rotation_not_90_func_full_layout( - image_page, textline_mask_tot, text_regions_p, regions_fully, slope_deskew) - - text_regions_p_1_n = resize_image(text_regions_p_1_n, text_regions_p.shape[0], text_regions_p.shape[1]) - textline_mask_tot_d = resize_image(textline_mask_tot_d, text_regions_p.shape[0], text_regions_p.shape[1]) - regions_fully_n = resize_image(regions_fully_n, text_regions_p.shape[0], text_regions_p.shape[1]) - if not self.tables: - regions_without_separators_d = (text_regions_p_1_n[:, :] == 1) * 1 - else: - text_regions_p_1_n = None - textline_mask_tot_d = None - regions_without_separators_d = None - if not self.tables: - regions_without_separators = (text_regions_p[:, :] == 1) * 1 - img_revised_tab = np.copy(text_regions_p[:, :]) - polygons_of_images = return_contours_of_interested_region(img_revised_tab, 5) - - self.logger.debug('exit run_boxes_full_layout') - #print("full inside 3", time.time()- t_full0) - return (polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, - regions_without_separators_d, regions_fully, regions_without_separators, - polygons_of_marginals, contours_tables) - - @staticmethod - def our_load_model(model_file): - if model_file.endswith('.h5') and Path(model_file[:-3]).exists(): - # prefer SavedModel over HDF5 format if it exists - model_file = model_file[:-3] - try: - model = load_model(model_file, compile=False) - except: - model = load_model(model_file, compile=False, custom_objects={ - "PatchEncoder": PatchEncoder, "Patches": Patches}) - return model - - def do_order_of_regions_with_model(self, contours_only_text_parent, contours_only_text_parent_h, text_regions_p): - y_len = text_regions_p.shape[0] - x_len = text_regions_p.shape[1] - - - img_poly = np.zeros((y_len,x_len), dtype='uint8') - img_poly[text_regions_p[:,:]==1] = 1 - img_poly[text_regions_p[:,:]==2] = 2 - img_poly[text_regions_p[:,:]==3] = 4 - img_poly[text_regions_p[:,:]==6] = 5 - - - #temp - sep_mask = (img_poly==5)*1 - sep_mask = sep_mask.astype('uint8') - sep_mask = cv2.erode(sep_mask, kernel=KERNEL, iterations=2) - img_poly[img_poly==5] = 0 - img_poly[sep_mask==1] = 5 - # - - img_header_and_sep = np.zeros((y_len,x_len), dtype='uint8') - if contours_only_text_parent_h: - _, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, _ = find_new_features_of_contours( - contours_only_text_parent_h) - for j in range(len(cy_main)): - img_header_and_sep[int(y_max_main[j]):int(y_max_main[j])+12, - int(x_min_main[j]):int(x_max_main[j])] = 1 - co_text_all = contours_only_text_parent + contours_only_text_parent_h - else: - co_text_all = contours_only_text_parent - - if not len(co_text_all): - return [], [] - - labels_con = np.zeros((int(y_len /6.), int(x_len/6.), len(co_text_all)), dtype=bool) - co_text_all = [(i/6).astype(int) for i in co_text_all] - for i in range(len(co_text_all)): - img = labels_con[:,:,i].astype(np.uint8) - - #img = cv2.resize(img, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) - - cv2.fillPoly(img, pts=[co_text_all[i]], color=(1,)) - labels_con[:,:,i] = img - - height1 =672#448 - width1 = 448#224 - - height2 =672#448 - width2= 448#224 - - height3 =672#448 - width3 = 448#224 - - labels_con = resize_image(labels_con.astype(np.uint8), height1, width1).astype(bool) - img_header_and_sep = resize_image(img_header_and_sep, height1, width1) - img_poly = resize_image(img_poly, height3, width3) - - - inference_bs = 3 - input_1 = np.zeros((inference_bs, height1, width1, 3)) - ordered = [list(range(len(co_text_all)))] - index_update = 0 - #print(labels_con.shape[2],"number of regions for reading order") - while index_update>=0: - ij_list = ordered.pop(index_update) - i = ij_list.pop(0) - - ante_list = [] - post_list = [] - tot_counter = 0 - batch = [] - for j in ij_list: - img1 = labels_con[:,:,i].astype(float) - img2 = labels_con[:,:,j].astype(float) - img1[img_poly==5] = 2 - img2[img_poly==5] = 2 - img1[img_header_and_sep==1] = 3 - img2[img_header_and_sep==1] = 3 - - input_1[len(batch), :, :, 0] = img1 / 3. - input_1[len(batch), :, :, 2] = img2 / 3. - input_1[len(batch), :, :, 1] = img_poly / 5. - - tot_counter += 1 - batch.append(j) - if tot_counter % inference_bs == 0 or tot_counter == len(ij_list): - y_pr = self.model_reading_order.predict(input_1 , verbose=0) - for jb, j in enumerate(batch): - if y_pr[jb][0]>=0.5: - post_list.append(j) - else: - ante_list.append(j) - batch = [] - - if len(ante_list): - ordered.insert(index_update, ante_list) - index_update += 1 - ordered.insert(index_update, [i]) - if len(post_list): - ordered.insert(index_update + 1, post_list) - - index_update = -1 - for index_next, ij_list in enumerate(ordered): - if len(ij_list) > 1: - index_update = index_next - break - - ordered = [i[0] for i in ordered] - region_ids = ['region_%04d' % i for i in range(len(co_text_all))] - return ordered, region_ids - - def return_start_and_end_of_common_text_of_textline_ocr(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.2*width) - - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: - print(len(peaks_real), 'len(peaks_real)') - - peaks_real = peaks_real[(peaks_realwidth1)] - - arg_sort = np.argsort(sum_smoothed[peaks_real]) - arg_sort4 =arg_sort[::-1][:4] - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - argsort_sorted = np.argsort(peaks_sort_4) - - first_4_sorted = peaks_sort_4[argsort_sorted] - y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] - #print(first_4_sorted,'first_4_sorted') - - arg_sortnew = np.argsort(y_4_sorted) - peaks_final =np.sort( first_4_sorted[arg_sortnew][2:] ) - - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peaks_final[0], peaks_final[0]], [0, height-1]) - #plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') - - return peaks_final[0], peaks_final[1] - else: - pass - - def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.06*width) - - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: - #print(len(peaks_real), 'len(peaks_real)') - - peaks_real = peaks_real[(peaks_realwidth1)] - - arg_max = np.argmax(sum_smoothed[peaks_real]) - peaks_final = peaks_real[arg_max] - - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peaks_final, peaks_final], [0, height-1]) - ##plt.plot([peaks_final[1], peaks_final[1]], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') - - return peaks_final - else: - return None - - def return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - self, peaks_real, sum_smoothed, start_split, end_split): - - peaks_real = peaks_real[(peaks_realstart_split)] - - arg_sort = np.argsort(sum_smoothed[peaks_real]) - arg_sort4 =arg_sort[::-1][:4] - peaks_sort_4 = peaks_real[arg_sort][::-1][:4] - argsort_sorted = np.argsort(peaks_sort_4) - - first_4_sorted = peaks_sort_4[argsort_sorted] - y_4_sorted = sum_smoothed[peaks_real][arg_sort4[argsort_sorted]] - #print(first_4_sorted,'first_4_sorted') - - arg_sortnew = np.argsort(y_4_sorted) - peaks_final =np.sort( first_4_sorted[arg_sortnew][3:] ) - return peaks_final[0] - - def return_start_and_end_of_common_text_of_textline_ocr_new(self, textline_image, ind_tot): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.15*width) - - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - mid = int(width/2.) - - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - if len(peaks_real)>70: - peak_start = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - peaks_real, sum_smoothed, width1, mid+2) - peak_end = self.return_start_and_end_of_common_text_of_textline_ocr_new_splitted( - peaks_real, sum_smoothed, mid-2, width2) - - #plt.figure(ind_tot) - #plt.imshow(textline_image) - #plt.plot([peak_start, peak_start], [0, height-1]) - #plt.plot([peak_end, peak_end], [0, height-1]) - #plt.savefig('./'+str(ind_tot)+'.png') - - return peak_start, peak_end - else: - pass - - def return_ocr_of_textline_without_common_section( - self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): - - if h2w_ratio > 0.05: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - else: - #width = np.shape(textline_image)[1] - #height = np.shape(textline_image)[0] - #common_window = int(0.3*width) - #width1 = int ( width/2. - common_window ) - #width2 = int ( width/2. + common_window ) - - split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section( - textline_image, ind_tot) - if split_point: - image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) - image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) - - #pixel_values1 = processor(image1, return_tensors="pt").pixel_values - #pixel_values2 = processor(image2, return_tensors="pt").pixel_values - - pixel_values_merged = processor([image1,image2], return_tensors="pt").pixel_values - generated_ids_merged = model_ocr.generate(pixel_values_merged.to(device)) - generated_text_merged = processor.batch_decode(generated_ids_merged, skip_special_tokens=True) - - #print(generated_text_merged,'generated_text_merged') - - #generated_ids1 = model_ocr.generate(pixel_values1.to(device)) - #generated_ids2 = model_ocr.generate(pixel_values2.to(device)) - - #generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] - #generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] - - #generated_text = generated_text1 + ' ' + generated_text2 - generated_text = generated_text_merged[0] + ' ' + generated_text_merged[1] - - #print(generated_text1,'generated_text1') - #print(generated_text2, 'generated_text2') - #print('########################################') - else: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - - #print(generated_text,'generated_text') - #print('########################################') - return generated_text - - def return_ocr_of_textline( - self, textline_image, model_ocr, processor, device, width_textline, h2w_ratio,ind_tot): - - if h2w_ratio > 0.05: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - else: - #width = np.shape(textline_image)[1] - #height = np.shape(textline_image)[0] - #common_window = int(0.3*width) - #width1 = int ( width/2. - common_window ) - #width2 = int ( width/2. + common_window ) - - try: - width1, width2 = self.return_start_and_end_of_common_text_of_textline_ocr_new(textline_image, ind_tot) - - image1 = textline_image[:, :width2,:]# image.crop((0, 0, width2, height)) - image2 = textline_image[:, width1:,:]#image.crop((width1, 0, width, height)) - - pixel_values1 = processor(image1, return_tensors="pt").pixel_values - pixel_values2 = processor(image2, return_tensors="pt").pixel_values - - generated_ids1 = model_ocr.generate(pixel_values1.to(device)) - generated_ids2 = model_ocr.generate(pixel_values2.to(device)) - - generated_text1 = processor.batch_decode(generated_ids1, skip_special_tokens=True)[0] - generated_text2 = processor.batch_decode(generated_ids2, skip_special_tokens=True)[0] - #print(generated_text1,'generated_text1') - #print(generated_text2, 'generated_text2') - #print('########################################') - - match = sq(None, generated_text1, generated_text2).find_longest_match( - 0, len(generated_text1), 0, len(generated_text2)) - generated_text = generated_text1 + generated_text2[match.b+match.size:] - except: - pixel_values = processor(textline_image, return_tensors="pt").pixel_values - generated_ids = model_ocr.generate(pixel_values.to(device)) - generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] - - return generated_text - - def return_textline_contour_with_added_box_coordinate(self, textline_contour, box_ind): - textline_contour[:,0] = textline_contour[:,0] + box_ind[2] - textline_contour[:,1] = textline_contour[:,1] + box_ind[0] - return textline_contour - - def return_list_of_contours_with_desired_order(self, ls_cons, sorted_indexes): - return [ls_cons[sorted_indexes[index]] for index in range(len(sorted_indexes))] - - def return_it_in_two_groups(self, x_differential): - split = [ind if x_differential[ind]!=x_differential[ind+1] else -1 - for ind in range(len(x_differential)-1)] - split_masked = list( np.array(split[:])[np.array(split[:])!=-1] ) - if 0 not in split_masked: - split_masked.insert(0, -1) - split_masked.append(len(x_differential)-1) - - split_masked = np.array(split_masked) +1 - - sums = [np.sum(x_differential[split_masked[ind]:split_masked[ind+1]]) - for ind in range(len(split_masked)-1)] - - indexes_to_bec_changed = [ind if (np.abs(sums[ind-1]) > np.abs(sums[ind]) and - np.abs(sums[ind+1]) > np.abs(sums[ind])) else -1 - for ind in range(1,len(sums)-1)] - indexes_to_bec_changed_filtered = np.array(indexes_to_bec_changed)[np.array(indexes_to_bec_changed)!=-1] - - x_differential_new = np.copy(x_differential) - for i in indexes_to_bec_changed_filtered: - i_slice = slice(split_masked[i], split_masked[i+1]) - x_differential_new[i_slice] = -1 * np.array(x_differential)[i_slice] - - return x_differential_new - - def dilate_textregions_contours_textline_version(self, all_found_textline_polygons): - #print(all_found_textline_polygons) - for j in range(len(all_found_textline_polygons)): - for ij in range(len(all_found_textline_polygons[j])): - con_ind = all_found_textline_polygons[j][ij] - area = cv2.contourArea(con_ind) - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_differential = gaussian_filter1d(x_differential, 0.1) - y_differential = gaussian_filter1d(y_differential, 0.1) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - x_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in x_differential] - y_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in y_differential] - - abs_diff=abs(abs(x_differential)- abs(y_differential) ) - - inc_x = np.zeros(len(x_differential)+1) - inc_y = np.zeros(len(x_differential)+1) - - if (y_max-y_min) <= (x_max-x_min): - dilation_m1 = round(area / (x_max-x_min) * 0.12) - else: - dilation_m1 = round(area / (y_max-y_min) * 0.12) - - if dilation_m1>8: - dilation_m1 = 8 - if dilation_m1<6: - dilation_m1 = 6 - #print(dilation_m1, 'dilation_m1') - dilation_m1 = 6 - dilation_m2 = int(dilation_m1/2.) +1 - - for i in range(len(x_differential)): - if abs_diff[i]==0: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]==0 and y_differential_mask_nonzeros[i]!=0: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]!=0 and y_differential_mask_nonzeros[i]==0: - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - - elif abs_diff[i]!=0 and abs_diff[i]>=3: - if abs(x_differential[i])>abs(y_differential[i]): - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - else: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - else: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - - inc_x[0] = inc_x[-1] - inc_y[0] = inc_y[-1] - - con_scaled = con_ind*1 - - con_scaled[:,0, 0] = con_ind[:,0,0] + np.array(inc_x)[:] - con_scaled[:,0, 1] = con_ind[:,0,1] + np.array(inc_y)[:] - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - area_scaled = cv2.contourArea(con_scaled.astype(np.int32)) - - con_ind = con_ind.astype(np.int32) - - results = [cv2.pointPolygonTest(con_ind, (con_scaled[ind,0, 0], con_scaled[ind,0, 1]), False) - for ind in range(len(con_scaled[:,0, 1])) ] - results = np.array(results) - #print(results,'results') - results[results==0] = 1 - - diff_result = np.diff(results) - - indices_2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==2] - indices_m2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==-2] - - if results[0]==1: - con_scaled[:indices_m2[0]+1,0, 1] = con_ind[:indices_m2[0]+1,0,1] - con_scaled[:indices_m2[0]+1,0, 0] = con_ind[:indices_m2[0]+1,0,0] - #indices_2 = indices_2[1:] - indices_m2 = indices_m2[1:] - - if len(indices_2)>len(indices_m2): - con_scaled[indices_2[-1]+1:,0, 1] = con_ind[indices_2[-1]+1:,0,1] - con_scaled[indices_2[-1]+1:,0, 0] = con_ind[indices_2[-1]+1:,0,0] - indices_2 = indices_2[:-1] - - for ii in range(len(indices_2)): - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 1] = con_scaled[indices_2[ii],0, 1] - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 0] = con_scaled[indices_2[ii],0, 0] - - all_found_textline_polygons[j][ij][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][ij][:,0,0] = con_scaled[:,0, 0] - return all_found_textline_polygons - - def dilate_textregions_contours(self, all_found_textline_polygons): - #print(all_found_textline_polygons) - for j in range(len(all_found_textline_polygons)): - con_ind = all_found_textline_polygons[j] - #print(len(con_ind[:,0,0]),'con_ind[:,0,0]') - area = cv2.contourArea(con_ind) - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_differential = gaussian_filter1d(x_differential, 0.1) - y_differential = gaussian_filter1d(y_differential, 0.1) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - x_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in x_differential] - y_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in y_differential] - - abs_diff=abs(abs(x_differential)- abs(y_differential) ) - - inc_x = np.zeros(len(x_differential)+1) - inc_y = np.zeros(len(x_differential)+1) - - if (y_max-y_min) <= (x_max-x_min): - dilation_m1 = round(area / (x_max-x_min) * 0.12) - else: - dilation_m1 = round(area / (y_max-y_min) * 0.12) - - if dilation_m1>8: - dilation_m1 = 8 - if dilation_m1<6: - dilation_m1 = 6 - #print(dilation_m1, 'dilation_m1') - dilation_m1 = 6 - dilation_m2 = int(dilation_m1/2.) +1 - - for i in range(len(x_differential)): - if abs_diff[i]==0: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]==0 and y_differential_mask_nonzeros[i]!=0: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]!=0 and y_differential_mask_nonzeros[i]==0: - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - - elif abs_diff[i]!=0 and abs_diff[i]>=3: - if abs(x_differential[i])>abs(y_differential[i]): - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - else: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - else: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - - inc_x[0] = inc_x[-1] - inc_y[0] = inc_y[-1] - - con_scaled = con_ind*1 - - con_scaled[:,0, 0] = con_ind[:,0,0] + np.array(inc_x)[:] - con_scaled[:,0, 1] = con_ind[:,0,1] + np.array(inc_y)[:] - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - area_scaled = cv2.contourArea(con_scaled.astype(np.int32)) - - con_ind = con_ind.astype(np.int32) - - results = [cv2.pointPolygonTest(con_ind, (con_scaled[ind,0, 0], con_scaled[ind,0, 1]), False) - for ind in range(len(con_scaled[:,0, 1])) ] - results = np.array(results) - #print(results,'results') - results[results==0] = 1 - - diff_result = np.diff(results) - indices_2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==2] - indices_m2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==-2] - - if results[0]==1: - con_scaled[:indices_m2[0]+1,0, 1] = con_ind[:indices_m2[0]+1,0,1] - con_scaled[:indices_m2[0]+1,0, 0] = con_ind[:indices_m2[0]+1,0,0] - #indices_2 = indices_2[1:] - indices_m2 = indices_m2[1:] - - if len(indices_2)>len(indices_m2): - con_scaled[indices_2[-1]+1:,0, 1] = con_ind[indices_2[-1]+1:,0,1] - con_scaled[indices_2[-1]+1:,0, 0] = con_ind[indices_2[-1]+1:,0,0] - indices_2 = indices_2[:-1] - - for ii in range(len(indices_2)): - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 1] = con_scaled[indices_2[ii],0, 1] - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 0] = con_scaled[indices_2[ii],0, 0] - - all_found_textline_polygons[j][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][:,0,0] = con_scaled[:,0, 0] - return all_found_textline_polygons - - def dilate_textline_contours(self, all_found_textline_polygons): - for j in range(len(all_found_textline_polygons)): - for ij in range(len(all_found_textline_polygons[j])): - con_ind = all_found_textline_polygons[j][ij] - area = cv2.contourArea(con_ind) - - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_differential = gaussian_filter1d(x_differential, 3) - y_differential = gaussian_filter1d(y_differential, 3) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - x_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in x_differential] - y_differential_mask_nonzeros = [ ind/abs(ind) if ind!=0 else ind for ind in y_differential] - - abs_diff=abs(abs(x_differential)- abs(y_differential) ) - - inc_x = np.zeros(len(x_differential)+1) - inc_y = np.zeros(len(x_differential)+1) - - if (y_max-y_min) <= (x_max-x_min): - dilation_m1 = round(area / (x_max-x_min) * 0.35) - else: - dilation_m1 = round(area / (y_max-y_min) * 0.35) - - if dilation_m1>12: - dilation_m1 = 12 - if dilation_m1<4: - dilation_m1 = 4 - #print(dilation_m1, 'dilation_m1') - dilation_m2 = int(dilation_m1/2.) +1 - - for i in range(len(x_differential)): - if abs_diff[i]==0: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]==0 and y_differential_mask_nonzeros[i]!=0: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - elif abs_diff[i]!=0 and x_differential_mask_nonzeros[i]!=0 and y_differential_mask_nonzeros[i]==0: - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - - elif abs_diff[i]!=0 and abs_diff[i]>=3: - if abs(x_differential[i])>abs(y_differential[i]): - inc_y[i+1] = dilation_m1*(x_differential_mask_nonzeros[i]) - else: - inc_x[i+1]= dilation_m1*(-1*y_differential_mask_nonzeros[i]) - else: - inc_x[i+1] = dilation_m2*(-1*y_differential_mask_nonzeros[i]) - inc_y[i+1] = dilation_m2*(x_differential_mask_nonzeros[i]) - - inc_x[0] = inc_x[-1] - inc_y[0] = inc_y[-1] - - con_scaled = con_ind*1 - - con_scaled[:,0, 0] = con_ind[:,0,0] + np.array(inc_x)[:] - con_scaled[:,0, 1] = con_ind[:,0,1] + np.array(inc_y)[:] - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - con_ind = con_ind.astype(np.int32) - - results = [cv2.pointPolygonTest(con_ind, (con_scaled[ind,0, 0], con_scaled[ind,0, 1]), False) - for ind in range(len(con_scaled[:,0, 1])) ] - results = np.array(results) - results[results==0] = 1 - - diff_result = np.diff(results) - - indices_2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==2] - indices_m2 = [ind for ind in range(len(diff_result)) if diff_result[ind]==-2] - - if results[0]==1: - con_scaled[:indices_m2[0]+1,0, 1] = con_ind[:indices_m2[0]+1,0,1] - con_scaled[:indices_m2[0]+1,0, 0] = con_ind[:indices_m2[0]+1,0,0] - indices_m2 = indices_m2[1:] - - if len(indices_2)>len(indices_m2): - con_scaled[indices_2[-1]+1:,0, 1] = con_ind[indices_2[-1]+1:,0,1] - con_scaled[indices_2[-1]+1:,0, 0] = con_ind[indices_2[-1]+1:,0,0] - indices_2 = indices_2[:-1] - - for ii in range(len(indices_2)): - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 1] = con_scaled[indices_2[ii],0, 1] - con_scaled[indices_2[ii]+1:indices_m2[ii]+1,0, 0] = con_scaled[indices_2[ii],0, 0] - - all_found_textline_polygons[j][ij][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][ij][:,0,0] = con_scaled[:,0, 0] - return all_found_textline_polygons - - def filter_contours_inside_a_bigger_one(self,contours, contours_d_ordered, image, marginal_cnts=None, type_contour="textregion"): - if type_contour=="textregion": - areas = [cv2.contourArea(contours[j]) for j in range(len(contours))] - area_tot = image.shape[0]*image.shape[1] - - M_main = [cv2.moments(contours[j]) - for j in range(len(contours))] - cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - - areas_ratio = np.array(areas)/ area_tot - contours_index_small = [ind for ind in range(len(contours)) if areas_ratio[ind] < 1e-3] - contours_index_big = [ind for ind in range(len(contours)) if areas_ratio[ind] >= 1e-3] - - #contours_> = [contours[ind] for ind in contours_index_big] - indexes_to_be_removed = [] - for ind_small in contours_index_small: - results = [cv2.pointPolygonTest(contours[ind], (cx_main[ind_small], cy_main[ind_small]), False) - for ind in contours_index_big] - if marginal_cnts: - results_marginal = [cv2.pointPolygonTest(marginal_cnts[ind], (cx_main[ind_small], cy_main[ind_small]), False) - for ind in range(len(marginal_cnts))] - results_marginal = np.array(results_marginal) - - if np.any(results_marginal==1): - indexes_to_be_removed.append(ind_small) - - results = np.array(results) - - if np.any(results==1): - indexes_to_be_removed.append(ind_small) - - if len(indexes_to_be_removed)>0: - indexes_to_be_removed = np.unique(indexes_to_be_removed) - indexes_to_be_removed = np.sort(indexes_to_be_removed)[::-1] - for ind in indexes_to_be_removed: - contours.pop(ind) - if len(contours_d_ordered)>0: - contours_d_ordered.pop(ind) - - return contours, contours_d_ordered - - else: - contours_txtline_of_all_textregions = [] - indexes_of_textline_tot = [] - index_textline_inside_textregion = [] - - for jj in range(len(contours)): - contours_txtline_of_all_textregions = contours_txtline_of_all_textregions + contours[jj] - - ind_textline_inside_tr = list(range(len(contours[jj]))) - index_textline_inside_textregion = index_textline_inside_textregion + ind_textline_inside_tr - ind_ins = [jj] * len(contours[jj]) - indexes_of_textline_tot = indexes_of_textline_tot + ind_ins - - M_main_tot = [cv2.moments(contours_txtline_of_all_textregions[j]) - for j in range(len(contours_txtline_of_all_textregions))] - cx_main_tot = [(M_main_tot[j]["m10"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - cy_main_tot = [(M_main_tot[j]["m01"] / (M_main_tot[j]["m00"] + 1e-32)) for j in range(len(M_main_tot))] - - areas_tot = [cv2.contourArea(con_ind) for con_ind in contours_txtline_of_all_textregions] - area_tot_tot = image.shape[0]*image.shape[1] - - textregion_index_to_del = [] - textline_in_textregion_index_to_del = [] - for ij in range(len(contours_txtline_of_all_textregions)): - args_all = list(np.array(range(len(contours_txtline_of_all_textregions)))) - args_all.pop(ij) - - areas_without = np.array(areas_tot)[args_all] - area_of_con_interest = areas_tot[ij] - - args_with_bigger_area = np.array(args_all)[areas_without > 1.5*area_of_con_interest] - - if len(args_with_bigger_area)>0: - results = [cv2.pointPolygonTest(contours_txtline_of_all_textregions[ind], (cx_main_tot[ij], cy_main_tot[ij]), False) - for ind in args_with_bigger_area ] - results = np.array(results) - if np.any(results==1): - #print(indexes_of_textline_tot[ij], index_textline_inside_textregion[ij]) - textregion_index_to_del.append(int(indexes_of_textline_tot[ij])) - textline_in_textregion_index_to_del.append(int(index_textline_inside_textregion[ij])) - #contours[int(indexes_of_textline_tot[ij])].pop(int(index_textline_inside_textregion[ij])) - - textregion_index_to_del = np.array(textregion_index_to_del) - textline_in_textregion_index_to_del = np.array(textline_in_textregion_index_to_del) - for ind_u_a_trs in np.unique(textregion_index_to_del): - textline_in_textregion_index_to_del_ind = textline_in_textregion_index_to_del[textregion_index_to_del==ind_u_a_trs] - textline_in_textregion_index_to_del_ind = np.sort(textline_in_textregion_index_to_del_ind)[::-1] - for ittrd in textline_in_textregion_index_to_del_ind: - contours[ind_u_a_trs].pop(ittrd) - - return contours - - def filter_contours_without_textline_inside( - self, contours,text_con_org, contours_textline, contours_only_text_parent_d_ordered, conf_contours_textregions): - ###contours_txtline_of_all_textregions = [] - ###for jj in range(len(contours_textline)): - ###contours_txtline_of_all_textregions = contours_txtline_of_all_textregions + contours_textline[jj] - - ###M_main_textline = [cv2.moments(contours_txtline_of_all_textregions[j]) - ### for j in range(len(contours_txtline_of_all_textregions))] - ###cx_main_textline = [(M_main_textline[j]["m10"] / (M_main_textline[j]["m00"] + 1e-32)) - ### for j in range(len(M_main_textline))] - ###cy_main_textline = [(M_main_textline[j]["m01"] / (M_main_textline[j]["m00"] + 1e-32)) - ### for j in range(len(M_main_textline))] - - ###M_main = [cv2.moments(contours[j]) for j in range(len(contours))] - ###cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - ###cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - - ###contours_with_textline = [] - ###for ind_tr, con_tr in enumerate(contours): - ###results = [cv2.pointPolygonTest(con_tr, (cx_main_textline[index_textline_con], cy_main_textline[index_textline_con]), False) - ### for index_textline_con in range(len(contours_txtline_of_all_textregions)) ] - ###results = np.array(results) - ###if np.any(results==1): - ###contours_with_textline.append(con_tr) - - textregion_index_to_del = [] - for index_textregion, textlines_textregion in enumerate(contours_textline): - if len(textlines_textregion)==0: - textregion_index_to_del.append(index_textregion) - - uniqe_args_trs = np.unique(textregion_index_to_del) - uniqe_args_trs_sorted = np.sort(uniqe_args_trs)[::-1] - - for ind_u_a_trs in uniqe_args_trs_sorted: - conf_contours_textregions.pop(ind_u_a_trs) - contours.pop(ind_u_a_trs) - contours_textline.pop(ind_u_a_trs) - text_con_org.pop(ind_u_a_trs) - if len(contours_only_text_parent_d_ordered) > 0: - contours_only_text_parent_d_ordered.pop(ind_u_a_trs) - - return contours, text_con_org, conf_contours_textregions, contours_textline, contours_only_text_parent_d_ordered, np.array(range(len(contours))) - - def dilate_textlines(self, all_found_textline_polygons): - for j in range(len(all_found_textline_polygons)): - for i in range(len(all_found_textline_polygons[j])): - con_ind = all_found_textline_polygons[j][i] - con_ind = con_ind.astype(float) - - x_differential = np.diff( con_ind[:,0,0]) - y_differential = np.diff( con_ind[:,0,1]) - - x_min = float(np.min( con_ind[:,0,0] )) - y_min = float(np.min( con_ind[:,0,1] )) - - x_max = float(np.max( con_ind[:,0,0] )) - y_max = float(np.max( con_ind[:,0,1] )) - - if (y_max - y_min) > (x_max - x_min) and (x_max - x_min)<70: - x_biger_than_x = np.abs(x_differential) > np.abs(y_differential) - mult = x_biger_than_x*x_differential - - arg_min_mult = np.argmin(mult) - arg_max_mult = np.argmax(mult) - - if y_differential[0]==0: - y_differential[0] = 0.1 - if y_differential[-1]==0: - y_differential[-1]= 0.1 - y_differential = [y_differential[ind] if y_differential[ind] != 0 - else 0.5 * (y_differential[ind-1] + y_differential[ind+1]) - for ind in range(len(y_differential))] - - if y_differential[0]==0.1: - y_differential[0] = y_differential[1] - if y_differential[-1]==0.1: - y_differential[-1] = y_differential[-2] - y_differential.append(y_differential[0]) - - y_differential = [-1 if y_differential[ind] < 0 else 1 - for ind in range(len(y_differential))] - y_differential = self.return_it_in_two_groups(y_differential) - y_differential = np.array(y_differential) - - con_scaled = con_ind*1 - con_scaled[:,0, 0] = con_ind[:,0,0] - 8*y_differential - con_scaled[arg_min_mult,0, 1] = con_ind[arg_min_mult,0,1] + 8 - con_scaled[arg_min_mult+1,0, 1] = con_ind[arg_min_mult+1,0,1] + 8 - - try: - con_scaled[arg_min_mult-1,0, 1] = con_ind[arg_min_mult-1,0,1] + 5 - con_scaled[arg_min_mult+2,0, 1] = con_ind[arg_min_mult+2,0,1] + 5 - except: - pass - - con_scaled[arg_max_mult,0, 1] = con_ind[arg_max_mult,0,1] - 8 - con_scaled[arg_max_mult+1,0, 1] = con_ind[arg_max_mult+1,0,1] - 8 - - try: - con_scaled[arg_max_mult-1,0, 1] = con_ind[arg_max_mult-1,0,1] - 5 - con_scaled[arg_max_mult+2,0, 1] = con_ind[arg_max_mult+2,0,1] - 5 - except: - pass - - else: - y_biger_than_x = np.abs(y_differential) > np.abs(x_differential) - mult = y_biger_than_x*y_differential - - arg_min_mult = np.argmin(mult) - arg_max_mult = np.argmax(mult) - - if x_differential[0]==0: - x_differential[0] = 0.1 - if x_differential[-1]==0: - x_differential[-1]= 0.1 - x_differential = [x_differential[ind] if x_differential[ind] != 0 - else 0.5 * (x_differential[ind-1] + x_differential[ind+1]) - for ind in range(len(x_differential))] - - if x_differential[0]==0.1: - x_differential[0] = x_differential[1] - if x_differential[-1]==0.1: - x_differential[-1] = x_differential[-2] - x_differential.append(x_differential[0]) - - x_differential = [-1 if x_differential[ind] < 0 else 1 - for ind in range(len(x_differential))] - x_differential = self.return_it_in_two_groups(x_differential) - x_differential = np.array(x_differential) - - con_scaled = con_ind*1 - con_scaled[:,0, 1] = con_ind[:,0,1] + 8*x_differential - con_scaled[arg_min_mult,0, 0] = con_ind[arg_min_mult,0,0] + 8 - con_scaled[arg_min_mult+1,0, 0] = con_ind[arg_min_mult+1,0,0] + 8 - - try: - con_scaled[arg_min_mult-1,0, 0] = con_ind[arg_min_mult-1,0,0] + 5 - con_scaled[arg_min_mult+2,0, 0] = con_ind[arg_min_mult+2,0,0] + 5 - except: - pass - - con_scaled[arg_max_mult,0, 0] = con_ind[arg_max_mult,0,0] - 8 - con_scaled[arg_max_mult+1,0, 0] = con_ind[arg_max_mult+1,0,0] - 8 - - try: - con_scaled[arg_max_mult-1,0, 0] = con_ind[arg_max_mult-1,0,0] - 5 - con_scaled[arg_max_mult+2,0, 0] = con_ind[arg_max_mult+2,0,0] - 5 - except: - pass - - con_scaled[:,0, 1][con_scaled[:,0, 1]<0] = 0 - con_scaled[:,0, 0][con_scaled[:,0, 0]<0] = 0 - - all_found_textline_polygons[j][i][:,0,1] = con_scaled[:,0, 1] - all_found_textline_polygons[j][i][:,0,0] = con_scaled[:,0, 0] - - return all_found_textline_polygons - - def delete_regions_without_textlines( - self, slopes, all_found_textline_polygons, boxes_text, txt_con_org, - contours_only_text_parent, index_by_text_par_con): - - slopes_rem = [] - all_found_textline_polygons_rem = [] - boxes_text_rem = [] - txt_con_org_rem = [] - contours_only_text_parent_rem = [] - index_by_text_par_con_rem = [] - - for i, ind_con in enumerate(all_found_textline_polygons): - if len(ind_con): - all_found_textline_polygons_rem.append(ind_con) - slopes_rem.append(slopes[i]) - boxes_text_rem.append(boxes_text[i]) - txt_con_org_rem.append(txt_con_org[i]) - contours_only_text_parent_rem.append(contours_only_text_parent[i]) - index_by_text_par_con_rem.append(index_by_text_par_con[i]) - - index_sort = np.argsort(index_by_text_par_con_rem) - indexes_new = np.array(range(len(index_by_text_par_con_rem))) - - index_by_text_par_con_rem_sort = [indexes_new[index_sort==j][0] - for j in range(len(index_by_text_par_con_rem))] - - return (slopes_rem, all_found_textline_polygons_rem, boxes_text_rem, txt_con_org_rem, - contours_only_text_parent_rem, index_by_text_par_con_rem_sort) - - def run(self, image_filename : Optional[str] = None, dir_in : Optional[str] = None, overwrite : bool = False): - """ - Get image and scales, then extract the page of scanned image - """ - self.logger.debug("enter run") - t0_tot = time.time() - - if dir_in: - self.ls_imgs = os.listdir(dir_in) - elif image_filename: - self.ls_imgs = [image_filename] - else: - raise ValueError("run requires either a single image filename or a directory") - - for img_filename in self.ls_imgs: - self.logger.info(img_filename) - t0 = time.time() - - self.reset_file_name_dir(os.path.join(dir_in or "", img_filename)) - #print("text region early -11 in %.1fs", time.time() - t0) - if os.path.exists(self.writer.output_filename): - if overwrite: - self.logger.warning("will overwrite existing output file '%s'", self.writer.output_filename) - else: - self.logger.warning("will skip input for existing output file '%s'", self.writer.output_filename) - continue - - pcgts = self.run_single() - self.logger.info("Job done in %.1fs", time.time() - t0) - #print("Job done in %.1fs" % (time.time() - t0)) - self.writer.write_pagexml(pcgts) - - if dir_in: - self.logger.info("All jobs done in %.1fs", time.time() - t0_tot) - print("all Job done in %.1fs", time.time() - t0_tot) - - def run_single(self): - t0 = time.time() - img_res, is_image_enhanced, num_col_classifier, num_column_is_classified = self.run_enhancement(self.light_version) - self.logger.info("Enhancing took %.1fs ", time.time() - t0) - if self.extract_only_images: - text_regions_p_1, erosion_hurts, polygons_lines_xml, polygons_of_images, image_page, page_coord, cont_page = \ - self.get_regions_light_v_extract_only_images(img_res, is_image_enhanced, num_col_classifier) - ocr_all_textlines = None - pcgts = self.writer.build_pagexml_no_full_layout( - [], page_coord, [], [], [], [], - polygons_of_images, [], [], [], [], [], - cont_page, [], [], ocr_all_textlines, []) - if self.plotter: - self.plotter.write_images_into_directory(polygons_of_images, image_page) - return pcgts - - if self.skip_layout_and_reading_order: - _ ,_, _, textline_mask_tot_ea, img_bin_light, _ = \ - self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier, - skip_layout_and_reading_order=self.skip_layout_and_reading_order) - - page_coord, image_page, textline_mask_tot_ea, img_bin_light, cont_page = \ - self.run_graphics_and_columns_without_layout(textline_mask_tot_ea, img_bin_light) - - - ##all_found_textline_polygons =self.scale_contours_new(textline_mask_tot_ea) - - cnt_clean_rot_raw, hir_on_cnt_clean_rot = return_contours_of_image(textline_mask_tot_ea) - all_found_textline_polygons = filter_contours_area_of_image( - textline_mask_tot_ea, cnt_clean_rot_raw, hir_on_cnt_clean_rot, max_area=1, min_area=0.00001) - - all_found_textline_polygons=[ all_found_textline_polygons ] - - all_found_textline_polygons = self.dilate_textregions_contours_textline_version( - all_found_textline_polygons) - all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( - all_found_textline_polygons, None, textline_mask_tot_ea, type_contour="textline") - - - order_text_new = [0] - slopes =[0] - id_of_texts_tot =['region_0001'] - - polygons_of_images = [] - slopes_marginals = [] - polygons_of_marginals = [] - all_found_textline_polygons_marginals = [] - all_box_coord_marginals = [] - polygons_lines_xml = [] - contours_tables = [] - ocr_all_textlines = None - conf_contours_textregions =None - pcgts = self.writer.build_pagexml_no_full_layout( - cont_page, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, page_coord, polygons_of_images, polygons_of_marginals, - all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) - return pcgts - - #print("text region early -1 in %.1fs", time.time() - t0) - t1 = time.time() - if self.light_version: - text_regions_p_1 ,erosion_hurts, polygons_lines_xml, textline_mask_tot_ea, img_bin_light, confidence_matrix = \ - self.get_regions_light_v(img_res, is_image_enhanced, num_col_classifier) - #print("text region early -2 in %.1fs", time.time() - t0) - - if num_col_classifier == 1 or num_col_classifier ==2: - if num_col_classifier == 1: - img_w_new = 1000 - else: - img_w_new = 1300 - img_h_new = img_w_new * textline_mask_tot_ea.shape[0] // textline_mask_tot_ea.shape[1] - - textline_mask_tot_ea_deskew = resize_image(textline_mask_tot_ea,img_h_new, img_w_new ) - - slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea_deskew) - else: - slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) - #print("text region early -2,5 in %.1fs", time.time() - t0) - #self.logger.info("Textregion detection took %.1fs ", time.time() - t1t) - num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ - text_regions_p_1, cont_page, table_prediction, textline_mask_tot_ea, img_bin_light = \ - self.run_graphics_and_columns_light(text_regions_p_1, textline_mask_tot_ea, - num_col_classifier, num_column_is_classified, erosion_hurts, img_bin_light) - #self.logger.info("run graphics %.1fs ", time.time() - t1t) - #print("text region early -3 in %.1fs", time.time() - t0) - textline_mask_tot_ea_org = np.copy(textline_mask_tot_ea) - #print("text region early -4 in %.1fs", time.time() - t0) - else: - text_regions_p_1 ,erosion_hurts, polygons_lines_xml = \ - self.get_regions_from_xy_2models(img_res, is_image_enhanced, - num_col_classifier) - self.logger.info("Textregion detection took %.1fs ", time.time() - t1) - confidence_matrix = np.zeros((text_regions_p_1.shape[:2])) - - t1 = time.time() - num_col, num_col_classifier, img_only_regions, page_coord, image_page, mask_images, mask_lines, \ - text_regions_p_1, cont_page, table_prediction = \ - self.run_graphics_and_columns(text_regions_p_1, num_col_classifier, num_column_is_classified, erosion_hurts) - self.logger.info("Graphics detection took %.1fs ", time.time() - t1) - #self.logger.info('cont_page %s', cont_page) - #plt.imshow(table_prediction) - #plt.show() - - if not num_col: - self.logger.info("No columns detected, outputting an empty PAGE-XML") - ocr_all_textlines = None - pcgts = self.writer.build_pagexml_no_full_layout( - [], page_coord, [], [], [], [], [], [], [], [], [], [], - cont_page, [], [], ocr_all_textlines, []) - return pcgts - - #print("text region early in %.1fs", time.time() - t0) - t1 = time.time() - if not self.light_version: - textline_mask_tot_ea = self.run_textline(image_page) - self.logger.info("textline detection took %.1fs", time.time() - t1) - t1 = time.time() - slope_deskew, slope_first = self.run_deskew(textline_mask_tot_ea) - self.logger.info("deskewing took %.1fs", time.time() - t1) - elif num_col_classifier in (1,2): - org_h_l_m = textline_mask_tot_ea.shape[0] - org_w_l_m = textline_mask_tot_ea.shape[1] - if num_col_classifier == 1: - img_w_new = 2000 - else: - img_w_new = 2400 - img_h_new = img_w_new * textline_mask_tot_ea.shape[0] // textline_mask_tot_ea.shape[1] - - image_page = resize_image(image_page,img_h_new, img_w_new ) - textline_mask_tot_ea = resize_image(textline_mask_tot_ea,img_h_new, img_w_new ) - mask_images = resize_image(mask_images,img_h_new, img_w_new ) - mask_lines = resize_image(mask_lines,img_h_new, img_w_new ) - text_regions_p_1 = resize_image(text_regions_p_1,img_h_new, img_w_new ) - table_prediction = resize_image(table_prediction,img_h_new, img_w_new ) - - textline_mask_tot, text_regions_p, image_page_rotated = \ - self.run_marginals(image_page, textline_mask_tot_ea, mask_images, mask_lines, - num_col_classifier, slope_deskew, text_regions_p_1, table_prediction) - - if self.light_version and num_col_classifier in (1,2): - image_page = resize_image(image_page,org_h_l_m, org_w_l_m ) - textline_mask_tot_ea = resize_image(textline_mask_tot_ea,org_h_l_m, org_w_l_m ) - text_regions_p = resize_image(text_regions_p,org_h_l_m, org_w_l_m ) - textline_mask_tot = resize_image(textline_mask_tot,org_h_l_m, org_w_l_m ) - text_regions_p_1 = resize_image(text_regions_p_1,org_h_l_m, org_w_l_m ) - table_prediction = resize_image(table_prediction,org_h_l_m, org_w_l_m ) - image_page_rotated = resize_image(image_page_rotated,org_h_l_m, org_w_l_m ) - - self.logger.info("detection of marginals took %.1fs", time.time() - t1) - #print("text region early 2 marginal in %.1fs", time.time() - t0) - ## birdan sora chock chakir - t1 = time.time() - if not self.full_layout: - polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, \ - boxes, boxes_d, polygons_of_marginals, contours_tables = \ - self.run_boxes_no_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, - num_col_classifier, table_prediction, erosion_hurts) - ###polygons_of_marginals = self.dilate_textregions_contours(polygons_of_marginals) - else: - polygons_of_images, img_revised_tab, text_regions_p_1_n, textline_mask_tot_d, regions_without_separators_d, \ - regions_fully, regions_without_separators, polygons_of_marginals, contours_tables = \ - self.run_boxes_full_layout(image_page, textline_mask_tot, text_regions_p, slope_deskew, - num_col_classifier, img_only_regions, table_prediction, erosion_hurts, - img_bin_light if self.light_version else None) - ###polygons_of_marginals = self.dilate_textregions_contours(polygons_of_marginals) - if self.light_version: - drop_label_in_full_layout = 4 - textline_mask_tot_ea_org[img_revised_tab==drop_label_in_full_layout] = 0 - - - text_only = ((img_revised_tab[:, :] == 1)) * 1 - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - text_only_d = ((text_regions_p_1_n[:, :] == 1)) * 1 - - #print("text region early 2 in %.1fs", time.time() - t0) - ###min_con_area = 0.000005 - contours_only_text, hir_on_text = return_contours_of_image(text_only) - contours_only_text_parent = return_parent_contours(contours_only_text, hir_on_text) - if len(contours_only_text_parent) > 0: - areas_cnt_text = np.array([cv2.contourArea(c) for c in contours_only_text_parent]) - areas_cnt_text = areas_cnt_text / float(text_only.shape[0] * text_only.shape[1]) - #self.logger.info('areas_cnt_text %s', areas_cnt_text) - contours_biggest = contours_only_text_parent[np.argmax(areas_cnt_text)] - contours_only_text_parent = [c for jz, c in enumerate(contours_only_text_parent) - if areas_cnt_text[jz] > MIN_AREA_REGION] - areas_cnt_text_parent = [area for area in areas_cnt_text if area > MIN_AREA_REGION] - index_con_parents = np.argsort(areas_cnt_text_parent) - - contours_only_text_parent = self.return_list_of_contours_with_desired_order( - contours_only_text_parent, index_con_parents) - - ##try: - ##contours_only_text_parent = \ - ##list(np.array(contours_only_text_parent,dtype=object)[index_con_parents]) - ##except: - ##contours_only_text_parent = \ - ##list(np.array(contours_only_text_parent,dtype=np.int32)[index_con_parents]) - ##areas_cnt_text_parent = list(np.array(areas_cnt_text_parent)[index_con_parents]) - areas_cnt_text_parent = self.return_list_of_contours_with_desired_order( - areas_cnt_text_parent, index_con_parents) - - cx_bigest_big, cy_biggest_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest]) - cx_bigest, cy_biggest, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent) - - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - contours_only_text_d, hir_on_text_d = return_contours_of_image(text_only_d) - contours_only_text_parent_d = return_parent_contours(contours_only_text_d, hir_on_text_d) - - areas_cnt_text_d = np.array([cv2.contourArea(c) for c in contours_only_text_parent_d]) - areas_cnt_text_d = areas_cnt_text_d / float(text_only_d.shape[0] * text_only_d.shape[1]) - - if len(areas_cnt_text_d)>0: - contours_biggest_d = contours_only_text_parent_d[np.argmax(areas_cnt_text_d)] - index_con_parents_d = np.argsort(areas_cnt_text_d) - contours_only_text_parent_d = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d, index_con_parents_d) - #try: - #contours_only_text_parent_d = \ - #list(np.array(contours_only_text_parent_d,dtype=object)[index_con_parents_d]) - #except: - #contours_only_text_parent_d = \ - #list(np.array(contours_only_text_parent_d,dtype=np.int32)[index_con_parents_d]) - #areas_cnt_text_d = list(np.array(areas_cnt_text_d)[index_con_parents_d]) - areas_cnt_text_d = self.return_list_of_contours_with_desired_order( - areas_cnt_text_d, index_con_parents_d) - - cx_bigest_d_big, cy_biggest_d_big, _, _, _, _, _ = find_new_features_of_contours([contours_biggest_d]) - cx_bigest_d, cy_biggest_d, _, _, _, _, _ = find_new_features_of_contours(contours_only_text_parent_d) - try: - if len(cx_bigest_d) >= 5: - cx_bigest_d_last5 = cx_bigest_d[-5:] - cy_biggest_d_last5 = cy_biggest_d[-5:] - dists_d = [math.sqrt((cx_bigest_big[0] - cx_bigest_d_last5[j]) ** 2 + - (cy_biggest_big[0] - cy_biggest_d_last5[j]) ** 2) - for j in range(len(cy_biggest_d_last5))] - ind_largest = len(cx_bigest_d) -5 + np.argmin(dists_d) - else: - cx_bigest_d_last5 = cx_bigest_d[-len(cx_bigest_d):] - cy_biggest_d_last5 = cy_biggest_d[-len(cx_bigest_d):] - dists_d = [math.sqrt((cx_bigest_big[0]-cx_bigest_d_last5[j])**2 + - (cy_biggest_big[0]-cy_biggest_d_last5[j])**2) - for j in range(len(cy_biggest_d_last5))] - ind_largest = len(cx_bigest_d) - len(cx_bigest_d) + np.argmin(dists_d) - - cx_bigest_d_big[0] = cx_bigest_d[ind_largest] - cy_biggest_d_big[0] = cy_biggest_d[ind_largest] - except Exception as why: - self.logger.error(why) - - (h, w) = text_only.shape[:2] - center = (w // 2.0, h // 2.0) - M = cv2.getRotationMatrix2D(center, slope_deskew, 1.0) - M_22 = np.array(M)[:2, :2] - p_big = np.dot(M_22, [cx_bigest_big, cy_biggest_big]) - x_diff = p_big[0] - cx_bigest_d_big - y_diff = p_big[1] - cy_biggest_d_big - - contours_only_text_parent_d_ordered = [] - for i in range(len(contours_only_text_parent)): - p = np.dot(M_22, [cx_bigest[i], cy_biggest[i]]) - p[0] = p[0] - x_diff[0] - p[1] = p[1] - y_diff[0] - dists = [math.sqrt((p[0] - cx_bigest_d[j]) ** 2 + - (p[1] - cy_biggest_d[j]) ** 2) - for j in range(len(cx_bigest_d))] - contours_only_text_parent_d_ordered.append(contours_only_text_parent_d[np.argmin(dists)]) - # img2=np.zeros((text_only.shape[0],text_only.shape[1],3)) - # img2=cv2.fillPoly(img2,pts=[contours_only_text_parent_d[np.argmin(dists)]] ,color=(1,1,1)) - # plt.imshow(img2[:,:,0]) - # plt.show() - else: - contours_only_text_parent_d_ordered = [] - contours_only_text_parent_d = [] - contours_only_text_parent = [] - - else: - contours_only_text_parent_d_ordered = [] - contours_only_text_parent_d = [] - #contours_only_text_parent = [] - if not len(contours_only_text_parent): - # stop early - empty_marginals = [[]] * len(polygons_of_marginals) - if self.full_layout: - pcgts = self.writer.build_pagexml_full_layout( - [], [], page_coord, [], [], [], [], [], [], - polygons_of_images, contours_tables, [], - polygons_of_marginals, empty_marginals, empty_marginals, [], [], [], - cont_page, polygons_lines_xml, [], [], []) - else: - pcgts = self.writer.build_pagexml_no_full_layout( - [], page_coord, [], [], [], [], - polygons_of_images, - polygons_of_marginals, empty_marginals, empty_marginals, [], [], - cont_page, polygons_lines_xml, contours_tables, [], []) - return pcgts - - - - #print("text region early 3 in %.1fs", time.time() - t0) - if self.light_version: - contours_only_text_parent = self.dilate_textregions_contours( - contours_only_text_parent) - contours_only_text_parent , contours_only_text_parent_d_ordered = self.filter_contours_inside_a_bigger_one( - contours_only_text_parent, contours_only_text_parent_d_ordered, text_only, marginal_cnts=polygons_of_marginals) - #print("text region early 3.5 in %.1fs", time.time() - t0) - txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( - contours_only_text_parent, self.image, slope_first, confidence_matrix, map=self.executor.map) - #txt_con_org = self.dilate_textregions_contours(txt_con_org) - #contours_only_text_parent = self.dilate_textregions_contours(contours_only_text_parent) - else: - txt_con_org , conf_contours_textregions = get_textregion_contours_in_org_image_light( - contours_only_text_parent, self.image, slope_first, confidence_matrix, map=self.executor.map) - #print("text region early 4 in %.1fs", time.time() - t0) - boxes_text, _ = get_text_region_boxes_by_given_contours(contours_only_text_parent) - boxes_marginals, _ = get_text_region_boxes_by_given_contours(polygons_of_marginals) - #print("text region early 5 in %.1fs", time.time() - t0) - ## birdan sora chock chakir - if not self.curved_line: - if self.light_version: - if self.textline_light: - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ - all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_light2( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea_org, - image_page_rotated, boxes_text, slope_deskew) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_light2( - polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_org, - image_page_rotated, boxes_marginals, slope_deskew) - - #slopes, all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con = \ - # self.delete_regions_without_textlines(slopes, all_found_textline_polygons, - # boxes_text, txt_con_org, contours_only_text_parent, index_by_text_par_con) - #slopes_marginals, all_found_textline_polygons_marginals, boxes_marginals, polygons_of_marginals, polygons_of_marginals, _ = \ - # self.delete_regions_without_textlines(slopes_marginals, all_found_textline_polygons_marginals, - # boxes_marginals, polygons_of_marginals, polygons_of_marginals, np.array(range(len(polygons_of_marginals)))) - #all_found_textline_polygons = self.dilate_textlines(all_found_textline_polygons) - #####all_found_textline_polygons = self.dilate_textline_contours(all_found_textline_polygons) - all_found_textline_polygons = self.dilate_textregions_contours_textline_version( - all_found_textline_polygons) - all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( - all_found_textline_polygons, None, textline_mask_tot_ea_org, type_contour="textline") - all_found_textline_polygons_marginals = self.dilate_textregions_contours_textline_version( - all_found_textline_polygons_marginals) - contours_only_text_parent, txt_con_org, conf_contours_textregions, all_found_textline_polygons, contours_only_text_parent_d_ordered, \ - index_by_text_par_con = self.filter_contours_without_textline_inside( - contours_only_text_parent, txt_con_org, all_found_textline_polygons, contours_only_text_parent_d_ordered, conf_contours_textregions) - else: - textline_mask_tot_ea = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1) - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, all_box_coord, \ - index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_light( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea, - image_page_rotated, boxes_text, slope_deskew) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_light( - polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, - image_page_rotated, boxes_marginals, slope_deskew) - #all_found_textline_polygons = self.filter_contours_inside_a_bigger_one( - # all_found_textline_polygons, textline_mask_tot_ea_org, type_contour="textline") - else: - textline_mask_tot_ea = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=1) - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ - all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea, - image_page_rotated, boxes_text, slope_deskew) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new( - polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea, - image_page_rotated, boxes_marginals, slope_deskew) - else: - scale_param = 1 - textline_mask_tot_ea_erode = cv2.erode(textline_mask_tot_ea, kernel=KERNEL, iterations=2) - all_found_textline_polygons, boxes_text, txt_con_org, contours_only_text_parent, \ - all_box_coord, index_by_text_par_con, slopes = self.get_slopes_and_deskew_new_curved( - txt_con_org, contours_only_text_parent, textline_mask_tot_ea_erode, - image_page_rotated, boxes_text, text_only, - num_col_classifier, scale_param, slope_deskew) - all_found_textline_polygons = small_textlines_to_parent_adherence2( - all_found_textline_polygons, textline_mask_tot_ea, num_col_classifier) - all_found_textline_polygons_marginals, boxes_marginals, _, polygons_of_marginals, \ - all_box_coord_marginals, _, slopes_marginals = self.get_slopes_and_deskew_new_curved( - polygons_of_marginals, polygons_of_marginals, textline_mask_tot_ea_erode, - image_page_rotated, boxes_marginals, text_only, - num_col_classifier, scale_param, slope_deskew) - all_found_textline_polygons_marginals = small_textlines_to_parent_adherence2( - all_found_textline_polygons_marginals, textline_mask_tot_ea, num_col_classifier) - - #print("text region early 6 in %.1fs", time.time() - t0) - if self.full_layout: - if np.abs(slope_deskew) >= SLOPE_THRESHOLD: - contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d_ordered, index_by_text_par_con) - #try: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=np.int32)[index_by_text_par_con]) - #except: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=object)[index_by_text_par_con]) - else: - #takes long timee - contours_only_text_parent_d_ordered = None - if self.light_version: - fun = check_any_text_region_in_model_one_is_main_or_header_light - else: - fun = check_any_text_region_in_model_one_is_main_or_header - text_regions_p, contours_only_text_parent, contours_only_text_parent_h, all_box_coord, all_box_coord_h, \ - all_found_textline_polygons, all_found_textline_polygons_h, slopes, slopes_h, \ - contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, \ - conf_contours_textregions, conf_contours_textregions_h = fun( - text_regions_p, regions_fully, contours_only_text_parent, - all_box_coord, all_found_textline_polygons, slopes, contours_only_text_parent_d_ordered, conf_contours_textregions) - - if self.plotter: - self.plotter.save_plot_of_layout(text_regions_p, image_page) - self.plotter.save_plot_of_layout_all(text_regions_p, image_page) - - pixel_img = 4 - polygons_of_drop_capitals = return_contours_of_interested_region_by_min_size(text_regions_p, pixel_img) - all_found_textline_polygons = adhere_drop_capital_region_into_corresponding_textline( - text_regions_p, polygons_of_drop_capitals, contours_only_text_parent, contours_only_text_parent_h, - all_box_coord, all_box_coord_h, all_found_textline_polygons, all_found_textline_polygons_h, - kernel=KERNEL, curved_line=self.curved_line, textline_light=self.textline_light) - - if not self.reading_order_machine_based: - pixel_seps = 6 - if not self.headers_off: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps, contours_only_text_parent_h) - else: - _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps, contours_only_text_parent_h_d_ordered) - elif self.headers_off: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - num_col, _, matrix_of_lines_ch, splitter_y_new, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps) - else: - _, _, matrix_of_lines_ch_d, splitter_y_new_d, _ = find_number_of_columns_in_document( - np.repeat(text_regions_p_1_n[:, :, np.newaxis], 3, axis=2), - num_col_classifier, self.tables, pixel_seps) - - if num_col_classifier >= 3: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - regions_without_separators = regions_without_separators.astype(np.uint8) - regions_without_separators = cv2.erode(regions_without_separators[:, :], KERNEL, iterations=6) - else: - regions_without_separators_d = regions_without_separators_d.astype(np.uint8) - regions_without_separators_d = cv2.erode(regions_without_separators_d[:, :], KERNEL, iterations=6) - - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - boxes, peaks_neg_tot_tables = return_boxes_of_images_by_order_of_reading_new( - splitter_y_new, regions_without_separators, matrix_of_lines_ch, - num_col_classifier, erosion_hurts, self.tables, self.right2left) - else: - boxes_d, peaks_neg_tot_tables_d = return_boxes_of_images_by_order_of_reading_new( - splitter_y_new_d, regions_without_separators_d, matrix_of_lines_ch_d, - num_col_classifier, erosion_hurts, self.tables, self.right2left) - - if self.plotter: - self.plotter.write_images_into_directory(polygons_of_images, image_page) - t_order = time.time() - - if self.full_layout: - if self.reading_order_machine_based: - order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( - contours_only_text_parent, contours_only_text_parent_h, text_regions_p) - else: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) - else: - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent_d_ordered, contours_only_text_parent_h_d_ordered, boxes_d, textline_mask_tot_d) - self.logger.info("detection of reading order took %.1fs", time.time() - t_order) - - if self.ocr: - ocr_all_textlines = [] - else: - ocr_all_textlines = None - pcgts = self.writer.build_pagexml_full_layout( - contours_only_text_parent, contours_only_text_parent_h, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, all_found_textline_polygons_h, all_box_coord, all_box_coord_h, - polygons_of_images, contours_tables, polygons_of_drop_capitals, polygons_of_marginals, - all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_h, slopes_marginals, - cont_page, polygons_lines_xml, ocr_all_textlines, conf_contours_textregions, conf_contours_textregions_h) - return pcgts - - contours_only_text_parent_h = None - if self.reading_order_machine_based: - order_text_new, id_of_texts_tot = self.do_order_of_regions_with_model( - contours_only_text_parent, contours_only_text_parent_h, text_regions_p) - else: - if np.abs(slope_deskew) < SLOPE_THRESHOLD: - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent, contours_only_text_parent_h, boxes, textline_mask_tot) - else: - contours_only_text_parent_d_ordered = self.return_list_of_contours_with_desired_order( - contours_only_text_parent_d_ordered, index_by_text_par_con) - #try: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=object)[index_by_text_par_con]) - #except: - #contours_only_text_parent_d_ordered = \ - #list(np.array(contours_only_text_parent_d_ordered, dtype=np.int32)[index_by_text_par_con]) - order_text_new, id_of_texts_tot = self.do_order_of_regions( - contours_only_text_parent_d_ordered, contours_only_text_parent_h, boxes_d, textline_mask_tot_d) - - if self.ocr: - device = cuda.get_current_device() - device.reset() - gc.collect() - model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) - device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") - torch.cuda.empty_cache() - model_ocr.to(device) - - ind_tot = 0 - #cv2.imwrite('./img_out.png', image_page) - ocr_all_textlines = [] - for indexing, ind_poly_first in enumerate(all_found_textline_polygons): - ocr_textline_in_textregion = [] - for indexing2, ind_poly in enumerate(ind_poly_first): - if not (self.textline_light or self.curved_line): - ind_poly = copy.deepcopy(ind_poly) - box_ind = all_box_coord[indexing] - #print(ind_poly,np.shape(ind_poly), 'ind_poly') - #print(box_ind) - ind_poly = self.return_textline_contour_with_added_box_coordinate(ind_poly, box_ind) - #print(ind_poly_copy) - ind_poly[ind_poly<0] = 0 - x, y, w, h = cv2.boundingRect(ind_poly) - #print(ind_poly_copy, np.shape(ind_poly_copy)) - #print(x, y, w, h, h/float(w),'ratio') - h2w_ratio = h/float(w) - mask_poly = np.zeros(image_page.shape) - if not self.light_version: - img_poly_on_img = np.copy(image_page) - else: - img_poly_on_img = np.copy(img_bin_light) - mask_poly = cv2.fillPoly(mask_poly, pts=[ind_poly], color=(1, 1, 1)) - - if self.textline_light: - mask_poly = cv2.dilate(mask_poly, KERNEL, iterations=1) - img_poly_on_img[:,:,0][mask_poly[:,:,0] ==0] = 255 - img_poly_on_img[:,:,1][mask_poly[:,:,0] ==0] = 255 - img_poly_on_img[:,:,2][mask_poly[:,:,0] ==0] = 255 - - img_croped = img_poly_on_img[y:y+h, x:x+w, :] - #cv2.imwrite('./extracted_lines/'+str(ind_tot)+'.jpg', img_croped) - text_ocr = self.return_ocr_of_textline_without_common_section(img_croped, model_ocr, processor, device, w, h2w_ratio, ind_tot) - ocr_textline_in_textregion.append(text_ocr) - ind_tot = ind_tot +1 - ocr_all_textlines.append(ocr_textline_in_textregion) - - else: - ocr_all_textlines = None - #print(ocr_all_textlines) - self.logger.info("detection of reading order took %.1fs", time.time() - t_order) - pcgts = self.writer.build_pagexml_no_full_layout( - txt_con_org, page_coord, order_text_new, id_of_texts_tot, - all_found_textline_polygons, all_box_coord, polygons_of_images, polygons_of_marginals, - all_found_textline_polygons_marginals, all_box_coord_marginals, slopes, slopes_marginals, - cont_page, polygons_lines_xml, contours_tables, ocr_all_textlines, conf_contours_textregions) - return pcgts - - -class Eynollah_ocr: - def __init__( - self, - dir_models, - dir_xmls=None, - dir_in=None, - dir_in_bin=None, - dir_out=None, - dir_out_image_text=None, - tr_ocr=False, - export_textline_images_and_text=False, - do_not_mask_with_textline_contour=False, - draw_texts_on_image=False, - prediction_with_both_of_rgb_and_bin=False, - logger=None, - ): - self.dir_in = dir_in - self.dir_in_bin = dir_in_bin - self.dir_out = dir_out - self.dir_xmls = dir_xmls - self.dir_models = dir_models - self.tr_ocr = tr_ocr - self.export_textline_images_and_text = export_textline_images_and_text - self.do_not_mask_with_textline_contour = do_not_mask_with_textline_contour - self.draw_texts_on_image = draw_texts_on_image - self.dir_out_image_text = dir_out_image_text - self.prediction_with_both_of_rgb_and_bin = prediction_with_both_of_rgb_and_bin - if tr_ocr: - self.processor = TrOCRProcessor.from_pretrained("microsoft/trocr-base-printed") - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - self.model_ocr_dir = dir_models + "/trocr_model_ens_of_3_checkpoints_201124" - self.model_ocr = VisionEncoderDecoderModel.from_pretrained(self.model_ocr_dir) - self.model_ocr.to(self.device) - - else: - self.model_ocr_dir = dir_models + "/model_step_1050000_ocr"#"/model_0_ocr_cnnrnn"#"/model_23_ocr_cnnrnn" - model_ocr = load_model(self.model_ocr_dir , compile=False) - - self.prediction_model = tf.keras.models.Model( - model_ocr.get_layer(name = "image").input, - model_ocr.get_layer(name = "dense2").output) - - - with open(os.path.join(self.model_ocr_dir, "characters_org.txt"),"r") as config_file: - characters = json.load(config_file) - - - AUTOTUNE = tf.data.AUTOTUNE - - # Mapping characters to integers. - char_to_num = StringLookup(vocabulary=list(characters), mask_token=None) - - # Mapping integers back to original characters. - self.num_to_char = StringLookup( - vocabulary=char_to_num.get_vocabulary(), mask_token=None, invert=True - ) - - def decode_batch_predictions(self, pred, max_len = 128): - # input_len is the product of the batch size and the - # number of time steps. - input_len = np.ones(pred.shape[0]) * pred.shape[1] - - # Decode CTC predictions using greedy search. - # decoded is a tuple with 2 elements. - decoded = tf.keras.backend.ctc_decode(pred, - input_length = input_len, - beam_width = 100) - # The outputs are in the first element of the tuple. - # Additionally, the first element is actually a list, - # therefore we take the first element of that list as well. - #print(decoded,'decoded') - decoded = decoded[0][0][:, :max_len] - - #print(decoded, decoded.shape,'decoded') - - output = [] - for d in decoded: - # Convert the predicted indices to the corresponding chars. - d = tf.strings.reduce_join(self.num_to_char(d)) - d = d.numpy().decode("utf-8") - output.append(d) - return output - - - def distortion_free_resize(self, image, img_size): - w, h = img_size - image = tf.image.resize(image, size=(h, w), preserve_aspect_ratio=True) - - # Check tha amount of padding needed to be done. - pad_height = h - tf.shape(image)[0] - pad_width = w - tf.shape(image)[1] - - # Only necessary if you want to do same amount of padding on both sides. - if pad_height % 2 != 0: - height = pad_height // 2 - pad_height_top = height + 1 - pad_height_bottom = height - else: - pad_height_top = pad_height_bottom = pad_height // 2 - - if pad_width % 2 != 0: - width = pad_width // 2 - pad_width_left = width + 1 - pad_width_right = width - else: - pad_width_left = pad_width_right = pad_width // 2 - - image = tf.pad( - image, - paddings=[ - [pad_height_top, pad_height_bottom], - [pad_width_left, pad_width_right], - [0, 0], - ], - ) - - image = tf.transpose(image, (1, 0, 2)) - image = tf.image.flip_left_right(image) - return image - - def return_start_and_end_of_common_text_of_textline_ocr_without_common_section(self, textline_image): - width = np.shape(textline_image)[1] - height = np.shape(textline_image)[0] - common_window = int(0.22*width) - - width1 = int ( width/2. - common_window ) - width2 = int ( width/2. + common_window ) - - img_sum = np.sum(textline_image[:,:,0], axis=0) - sum_smoothed = gaussian_filter1d(img_sum, 3) - - peaks_real, _ = find_peaks(sum_smoothed, height=0) - - if len(peaks_real)>35: - - #peaks_real = peaks_real[(peaks_realwidth1)] - argsort = np.argsort(sum_smoothed[peaks_real])[::-1] - peaks_real_top_six = peaks_real[argsort[:6]] - midpoint = textline_image.shape[1] / 2. - arg_closest = np.argmin(np.abs(peaks_real_top_six - midpoint)) - - #arg_max = np.argmax(sum_smoothed[peaks_real]) - - peaks_final = peaks_real_top_six[arg_closest]#peaks_real[arg_max] - - return peaks_final - else: - return None - - # Function to fit text inside the given area - def fit_text_single_line(self, draw, text, font_path, max_width, max_height): - initial_font_size = 50 - font_size = initial_font_size - while font_size > 10: # Minimum font size - font = ImageFont.truetype(font_path, font_size) - text_bbox = draw.textbbox((0, 0), text, font=font) # Get text bounding box - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] - - if text_width <= max_width and text_height <= max_height: - return font # Return the best-fitting font - - font_size -= 2 # Reduce font size and retry - - return ImageFont.truetype(font_path, 10) # Smallest font fallback - - def return_textlines_split_if_needed(self, textline_image, textline_image_bin): - - split_point = self.return_start_and_end_of_common_text_of_textline_ocr_without_common_section(textline_image) - if split_point: - image1 = textline_image[:, :split_point,:]# image.crop((0, 0, width2, height)) - image2 = textline_image[:, split_point:,:]#image.crop((width1, 0, width, height)) - if self.prediction_with_both_of_rgb_and_bin: - image1_bin = textline_image_bin[:, :split_point,:]# image.crop((0, 0, width2, height)) - image2_bin = textline_image_bin[:, split_point:,:]#image.crop((width1, 0, width, height)) - return [image1, image2], [image1_bin, image2_bin] - else: - return [image1, image2], None - else: - return None, None - def preprocess_and_resize_image_for_ocrcnn_model(self, img, image_height, image_width): - ratio = image_height /float(img.shape[0]) - w_ratio = int(ratio * img.shape[1]) - - if w_ratio <= image_width: - width_new = w_ratio - else: - width_new = image_width - - if width_new == 0: - width_new = img.shape[1] - - ##if width_new+32 >= image_width: - ##width_new = width_new - 32 - - ###patch_zero = np.zeros((32, 32, 3))#+255 - ###patch_zero[9:19,8:18,:] = 0 - - - img = resize_image(img, image_height, width_new) - img_fin = np.ones((image_height, image_width, 3))*255 - ###img_fin[:,:32,:] = patch_zero[:,:,:] - ###img_fin[:,32:32+width_new,:] = img[:,:,:] - img_fin[:,:width_new,:] = img[:,:,:] - img_fin = img_fin / 255. - return img_fin - - def run(self): - ls_imgs = os.listdir(self.dir_in) - - if self.tr_ocr: - b_s = 2 - for ind_img in ls_imgs: - t0 = time.time() - file_name = ind_img.split('.')[0] - dir_img = os.path.join(self.dir_in, ind_img) - dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') - out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') - img = cv2.imread(dir_img) - - ##file_name = Path(dir_xmls).stem - tree1 = ET.parse(dir_xml, parser = ET.XMLParser(encoding="utf-8")) - root1=tree1.getroot() - alltags=[elem.tag for elem in root1.iter()] - link=alltags[0].split('}')[0]+'}' - - name_space = alltags[0].split('}')[0] - name_space = name_space.split('{')[1] - - region_tags=np.unique([x for x in alltags if x.endswith('TextRegion')]) - - - - cropped_lines = [] - cropped_lines_region_indexer = [] - cropped_lines_meging_indexing = [] - - indexer_text_region = 0 - for nn in root1.iter(region_tags): - for child_textregion in nn: - if child_textregion.tag.endswith("TextLine"): - - for child_textlines in child_textregion: - if child_textlines.tag.endswith("Coords"): - cropped_lines_region_indexer.append(indexer_text_region) - p_h=child_textlines.attrib['points'].split(' ') - textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) - x,y,w,h = cv2.boundingRect(textline_coords) - - h2w_ratio = h/float(w) - - img_poly_on_img = np.copy(img) - mask_poly = np.zeros(img.shape) - mask_poly = cv2.fillPoly(mask_poly, pts=[textline_coords], color=(1, 1, 1)) - - mask_poly = mask_poly[y:y+h, x:x+w, :] - img_crop = img_poly_on_img[y:y+h, x:x+w, :] - img_crop[mask_poly==0] = 255 - - if h2w_ratio > 0.1: - cropped_lines.append(img_crop) - cropped_lines_meging_indexing.append(0) - else: - splited_images, _ = self.return_textlines_split_if_needed(img_crop, None) - #print(splited_images) - if splited_images: - cropped_lines.append(splited_images[0]) - cropped_lines_meging_indexing.append(1) - cropped_lines.append(splited_images[1]) - cropped_lines_meging_indexing.append(-1) - else: - cropped_lines.append(img_crop) - cropped_lines_meging_indexing.append(0) - indexer_text_region = indexer_text_region +1 - - - extracted_texts = [] - n_iterations = math.ceil(len(cropped_lines) / b_s) - - for i in range(n_iterations): - if i==(n_iterations-1): - n_start = i*b_s - imgs = cropped_lines[n_start:] - else: - n_start = i*b_s - n_end = (i+1)*b_s - imgs = cropped_lines[n_start:n_end] - pixel_values_merged = self.processor(imgs, return_tensors="pt").pixel_values - generated_ids_merged = self.model_ocr.generate(pixel_values_merged.to(self.device)) - generated_text_merged = self.processor.batch_decode(generated_ids_merged, skip_special_tokens=True) - - extracted_texts = extracted_texts + generated_text_merged - - extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] - - extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] - #print(extracted_texts_merged, len(extracted_texts_merged)) - - unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - - #print(len(unique_cropped_lines_region_indexer), 'unique_cropped_lines_region_indexer') - text_by_textregion = [] - for ind in unique_cropped_lines_region_indexer: - extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] - - text_by_textregion.append(" ".join(extracted_texts_merged_un)) - - #print(len(text_by_textregion) , indexer_text_region, "text_by_textregion") - - - #print(time.time() - t0 ,'elapsed time') - - - indexer = 0 - indexer_textregion = 0 - for nn in root1.iter(region_tags): - text_subelement_textregion = ET.SubElement(nn, 'TextEquiv') - unicode_textregion = ET.SubElement(text_subelement_textregion, 'Unicode') - - - has_textline = False - for child_textregion in nn: - if child_textregion.tag.endswith("TextLine"): - text_subelement = ET.SubElement(child_textregion, 'TextEquiv') - unicode_textline = ET.SubElement(text_subelement, 'Unicode') - unicode_textline.text = extracted_texts_merged[indexer] - indexer = indexer + 1 - has_textline = True - if has_textline: - unicode_textregion.text = text_by_textregion[indexer_textregion] - indexer_textregion = indexer_textregion + 1 - - - - ET.register_namespace("",name_space) - tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) - #print("Job done in %.1fs", time.time() - t0) - else: - max_len = 512 - padding_token = 299 - image_width = 512#max_len * 4 - image_height = 32 - b_s = 8 - - - img_size=(image_width, image_height) - - for ind_img in ls_imgs: - t0 = time.time() - file_name = ind_img.split('.')[0] - dir_img = os.path.join(self.dir_in, ind_img) - dir_xml = os.path.join(self.dir_xmls, file_name+'.xml') - out_file_ocr = os.path.join(self.dir_out, file_name+'.xml') - img = cv2.imread(dir_img) - if self.prediction_with_both_of_rgb_and_bin: - cropped_lines_bin = [] - dir_img_bin = os.path.join(self.dir_in_bin, file_name+'.png') - img_bin = cv2.imread(dir_img_bin) - - if self.draw_texts_on_image: - out_image_with_text = os.path.join(self.dir_out_image_text, file_name+'.png') - image_text = Image.new("RGB", (img.shape[1], img.shape[0]), "white") - draw = ImageDraw.Draw(image_text) - total_bb_coordinates = [] - - tree1 = ET.parse(dir_xml, parser = ET.XMLParser(encoding="utf-8")) - root1=tree1.getroot() - alltags=[elem.tag for elem in root1.iter()] - link=alltags[0].split('}')[0]+'}' - - name_space = alltags[0].split('}')[0] - name_space = name_space.split('{')[1] - - region_tags=np.unique([x for x in alltags if x.endswith('TextRegion')]) - - cropped_lines = [] - cropped_lines_region_indexer = [] - cropped_lines_meging_indexing = [] - - tinl = time.time() - indexer_text_region = 0 - indexer_textlines = 0 - for nn in root1.iter(region_tags): - for child_textregion in nn: - if child_textregion.tag.endswith("TextLine"): - for child_textlines in child_textregion: - if child_textlines.tag.endswith("Coords"): - cropped_lines_region_indexer.append(indexer_text_region) - p_h=child_textlines.attrib['points'].split(' ') - textline_coords = np.array( [ [ int(x.split(',')[0]) , int(x.split(',')[1]) ] for x in p_h] ) - - x,y,w,h = cv2.boundingRect(textline_coords) - - if self.draw_texts_on_image: - total_bb_coordinates.append([x,y,w,h]) - - w_scaled = w * image_height/float(h) - - img_poly_on_img = np.copy(img) - if self.prediction_with_both_of_rgb_and_bin: - img_poly_on_img_bin = np.copy(img_bin) - img_crop_bin = img_poly_on_img_bin[y:y+h, x:x+w, :] - - mask_poly = np.zeros(img.shape) - mask_poly = cv2.fillPoly(mask_poly, pts=[textline_coords], color=(1, 1, 1)) - - mask_poly = mask_poly[y:y+h, x:x+w, :] - img_crop = img_poly_on_img[y:y+h, x:x+w, :] - if not self.do_not_mask_with_textline_contour: - img_crop[mask_poly==0] = 255 - if self.prediction_with_both_of_rgb_and_bin: - img_crop_bin[mask_poly==0] = 255 - - if not self.export_textline_images_and_text: - if w_scaled < 1.5*image_width: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(0) - if self.prediction_with_both_of_rgb_and_bin: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) - cropped_lines_bin.append(img_fin) - else: - if self.prediction_with_both_of_rgb_and_bin: - splited_images, splited_images_bin = self.return_textlines_split_if_needed(img_crop, img_crop_bin) - else: - splited_images, splited_images_bin = self.return_textlines_split_if_needed(img_crop, None) - if splited_images: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images[0], image_height, image_width) - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(1) - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images[1], image_height, image_width) - - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(-1) - - if self.prediction_with_both_of_rgb_and_bin: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[0], image_height, image_width) - cropped_lines_bin.append(img_fin) - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(splited_images_bin[1], image_height, image_width) - cropped_lines_bin.append(img_fin) - - else: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop, image_height, image_width) - cropped_lines.append(img_fin) - cropped_lines_meging_indexing.append(0) - - if self.prediction_with_both_of_rgb_and_bin: - img_fin = self.preprocess_and_resize_image_for_ocrcnn_model(img_crop_bin, image_height, image_width) - cropped_lines_bin.append(img_fin) - - if self.export_textline_images_and_text: - if child_textlines.tag.endswith("TextEquiv"): - for cheild_text in child_textlines: - if cheild_text.tag.endswith("Unicode"): - textline_text = cheild_text.text - if textline_text: - with open(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.txt'), 'w') as text_file: - text_file.write(textline_text) - - cv2.imwrite(os.path.join(self.dir_out, file_name+'_line_'+str(indexer_textlines)+'.png'), img_crop ) - - indexer_textlines+=1 - - if not self.export_textline_images_and_text: - indexer_text_region = indexer_text_region +1 - - if not self.export_textline_images_and_text: - extracted_texts = [] - - n_iterations = math.ceil(len(cropped_lines) / b_s) - - for i in range(n_iterations): - if i==(n_iterations-1): - n_start = i*b_s - imgs = cropped_lines[n_start:] - imgs = np.array(imgs) - imgs = imgs.reshape(imgs.shape[0], image_height, image_width, 3) - if self.prediction_with_both_of_rgb_and_bin: - imgs_bin = cropped_lines_bin[n_start:] - imgs_bin = np.array(imgs_bin) - imgs_bin = imgs_bin.reshape(imgs_bin.shape[0], image_height, image_width, 3) - else: - n_start = i*b_s - n_end = (i+1)*b_s - imgs = cropped_lines[n_start:n_end] - imgs = np.array(imgs).reshape(b_s, image_height, image_width, 3) - - if self.prediction_with_both_of_rgb_and_bin: - imgs_bin = cropped_lines_bin[n_start:n_end] - imgs_bin = np.array(imgs_bin).reshape(b_s, image_height, image_width, 3) - - - preds = self.prediction_model.predict(imgs, verbose=0) - if self.prediction_with_both_of_rgb_and_bin: - preds_bin = self.prediction_model.predict(imgs_bin, verbose=0) - preds = (preds + preds_bin) / 2. - - pred_texts = self.decode_batch_predictions(preds) - - for ib in range(imgs.shape[0]): - pred_texts_ib = pred_texts[ib].replace("[UNK]", "") - extracted_texts.append(pred_texts_ib) - - extracted_texts_merged = [extracted_texts[ind] if cropped_lines_meging_indexing[ind]==0 else extracted_texts[ind]+" "+extracted_texts[ind+1] if cropped_lines_meging_indexing[ind]==1 else None for ind in range(len(cropped_lines_meging_indexing))] - - extracted_texts_merged = [ind for ind in extracted_texts_merged if ind is not None] - unique_cropped_lines_region_indexer = np.unique(cropped_lines_region_indexer) - - - if self.draw_texts_on_image: - - font_path = "NotoSans-Regular.ttf" # Make sure this file exists! - font = ImageFont.truetype(font_path, 40) - - for indexer_text, bb_ind in enumerate(total_bb_coordinates): - - - x_bb = bb_ind[0] - y_bb = bb_ind[1] - w_bb = bb_ind[2] - h_bb = bb_ind[3] - - font = self.fit_text_single_line(draw, extracted_texts_merged[indexer_text], font_path, w_bb, int(h_bb*0.4) ) - - ##draw.rectangle([x_bb, y_bb, x_bb + w_bb, y_bb + h_bb], outline="red", width=2) - - text_bbox = draw.textbbox((0, 0), extracted_texts_merged[indexer_text], font=font) - text_width = text_bbox[2] - text_bbox[0] - text_height = text_bbox[3] - text_bbox[1] - - text_x = x_bb + (w_bb - text_width) // 2 # Center horizontally - text_y = y_bb + (h_bb - text_height) // 2 # Center vertically - - # Draw the text - draw.text((text_x, text_y), extracted_texts_merged[indexer_text], fill="black", font=font) - image_text.save(out_image_with_text) - - text_by_textregion = [] - for ind in unique_cropped_lines_region_indexer: - extracted_texts_merged_un = np.array(extracted_texts_merged)[np.array(cropped_lines_region_indexer)==ind] - text_by_textregion.append("".join(extracted_texts_merged_un)) - - indexer = 0 - indexer_textregion = 0 - for nn in root1.iter(region_tags): - - is_textregion_text = False - for childtest in nn: - if childtest.tag.endswith("TextEquiv"): - is_textregion_text = True - - if not is_textregion_text: - text_subelement_textregion = ET.SubElement(nn, 'TextEquiv') - unicode_textregion = ET.SubElement(text_subelement_textregion, 'Unicode') - - - has_textline = False - for child_textregion in nn: - if child_textregion.tag.endswith("TextLine"): - - is_textline_text = False - for childtest2 in child_textregion: - if childtest2.tag.endswith("TextEquiv"): - is_textline_text = True - - - if not is_textline_text: - text_subelement = ET.SubElement(child_textregion, 'TextEquiv') - unicode_textline = ET.SubElement(text_subelement, 'Unicode') - unicode_textline.text = extracted_texts_merged[indexer] - else: - for childtest3 in child_textregion: - if childtest3.tag.endswith("TextEquiv"): - for child_uc in childtest3: - if child_uc.tag.endswith("Unicode"): - child_uc.text = extracted_texts_merged[indexer] - - indexer = indexer + 1 - has_textline = True - if has_textline: - if is_textregion_text: - for child4 in nn: - if child4.tag.endswith("TextEquiv"): - for childtr_uc in child4: - if childtr_uc.tag.endswith("Unicode"): - childtr_uc.text = text_by_textregion[indexer_textregion] - else: - unicode_textregion.text = text_by_textregion[indexer_textregion] - indexer_textregion = indexer_textregion + 1 - - ET.register_namespace("",name_space) - tree1.write(out_file_ocr,xml_declaration=True,method='xml',encoding="utf8",default_namespace=None) - #print("Job done in %.1fs", time.time() - t0) diff --git a/src/eynollah/ocrd-tool.json b/src/eynollah/ocrd-tool.json deleted file mode 100644 index e972ec8..0000000 --- a/src/eynollah/ocrd-tool.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "version": "0.4.0", - "git_url": "https://github.com/qurator-spk/eynollah", - "dockerhub": "ocrd/eynollah", - "tools": { - "ocrd-eynollah-segment": { - "executable": "ocrd-eynollah-segment", - "categories": ["Layout analysis"], - "description": "Segment page into regions and lines and do reading order detection with eynollah", - "input_file_grp_cardinality": 1, - "output_file_grp_cardinality": 1, - "steps": ["layout/segmentation/region", "layout/segmentation/line"], - "parameters": { - "models": { - "type": "string", - "format": "uri", - "content-type": "text/directory", - "cacheable": true, - "description": "Directory containing models to be used (See https://qurator-data.de/eynollah)", - "required": true - }, - "dpi": { - "type": "number", - "format": "float", - "description": "pixel density in dots per inch (overrides any meta-data in the images); ignored if <= 0 (with fall-back 230)", - "default": 0 - }, - "full_layout": { - "type": "boolean", - "default": true, - "description": "Try to detect all element subtypes, including drop-caps and headings" - }, - "light_version": { - "type": "boolean", - "default": true, - "description": "Try to detect all element subtypes in light version (faster+simpler method for main region detection and deskewing)" - }, - "textline_light": { - "type": "boolean", - "default": true, - "description": "Light version need textline light" - }, - "tables": { - "type": "boolean", - "default": false, - "description": "Try to detect table regions" - }, - "curved_line": { - "type": "boolean", - "default": false, - "description": "try to return contour of textlines instead of just rectangle bounding box. Needs more processing time" - }, - "ignore_page_extraction": { - "type": "boolean", - "default": false, - "description": "if this parameter set to true, this tool would ignore page extraction" - }, - "allow_scaling": { - "type": "boolean", - "default": false, - "description": "check the resolution against the number of detected columns and if needed, scale the image up or down during layout detection (heuristic to improve quality and performance)" - }, - "allow_enhancement": { - "type": "boolean", - "default": false, - "description": "if this parameter set to true, this tool would check that input image need resizing and enhancement or not." - }, - "textline_light": { - "type": "boolean", - "default": false, - "description": "if this parameter set to true, this tool will try to return contoure of textlines instead of rectangle bounding box of textline with a faster method." - }, - "right_to_left": { - "type": "boolean", - "default": false, - "description": "if this parameter set to true, this tool will extract right-to-left reading order." - }, - "headers_off": { - "type": "boolean", - "default": false, - "description": "ignore the special role of headings during reading order detection" - } - }, - "resources": [ - { - "description": "models for eynollah (TensorFlow SavedModel format)", - "url": "https://github.com/qurator-spk/eynollah/releases/download/v0.3.1/models_eynollah.tar.gz", - "name": "default", - "size": 1894627041, - "type": "archive", - "path_in_archive": "models_eynollah" - } - ] - }, - "ocrd-sbb-binarize": { - "executable": "ocrd-sbb-binarize", - "description": "Pixelwise binarization with selectional auto-encoders in Keras", - "categories": ["Image preprocessing"], - "steps": ["preprocessing/optimization/binarization"], - "input_file_grp_cardinality": 1, - "output_file_grp_cardinality": 1, - "parameters": { - "operation_level": { - "type": "string", - "enum": ["page", "region"], - "default": "page", - "description": "PAGE XML hierarchy level to operate on" - }, - "model": { - "description": "Directory containing HDF5 or SavedModel/ProtoBuf models. Can be an absolute path or a path relative to the OCR-D resource location, the current working directory or the $SBB_BINARIZE_DATA environment variable (if set)", - "type": "string", - "format": "uri", - "content-type": "text/directory", - "required": true - } - }, - "resources": [ - { - "url": "https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2020_01_16.zip", - "name": "default", - "type": "archive", - "path_in_archive": "saved_model_2020_01_16", - "size": 563147331, - "description": "default models provided by github.com/qurator-spk (SavedModel format)" - }, - { - "url": "https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip", - "name": "default-2021-03-09", - "type": "archive", - "path_in_archive": ".", - "size": 133230419, - "description": "updated default models provided by github.com/qurator-spk (SavedModel format)" - } - ] - } - } -} diff --git a/src/eynollah/ocrd_cli_binarization.py b/src/eynollah/ocrd_cli_binarization.py deleted file mode 100644 index 848bbac..0000000 --- a/src/eynollah/ocrd_cli_binarization.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Optional - -from PIL import Image -import numpy as np -import cv2 -from click import command - -from ocrd import Processor, OcrdPageResult, OcrdPageResultImage -from ocrd_models.ocrd_page import OcrdPage, AlternativeImageType -from ocrd.decorators import ocrd_cli_options, ocrd_cli_wrap_processor - -from .sbb_binarize import SbbBinarizer - - -def cv2pil(img): - return Image.fromarray(img.astype('uint8')) - -def pil2cv(img): - # from ocrd/workspace.py - color_conversion = cv2.COLOR_GRAY2BGR if img.mode in ('1', 'L') else cv2.COLOR_RGB2BGR - pil_as_np_array = np.array(img).astype('uint8') if img.mode == '1' else np.array(img) - return cv2.cvtColor(pil_as_np_array, color_conversion) - -class SbbBinarizeProcessor(Processor): - # already employs GPU (without singleton process atm) - max_workers = 1 - - @property - def executable(self): - return 'ocrd-sbb-binarize' - - def setup(self): - """ - Set up the model prior to processing. - """ - # resolve relative path via OCR-D ResourceManager - model_path = self.resolve_resource(self.parameter['model']) - self.binarizer = SbbBinarizer(model_dir=model_path, logger=self.logger) - - def process_page_pcgts(self, *input_pcgts: Optional[OcrdPage], page_id: Optional[str] = None) -> OcrdPageResult: - """ - Binarize images with sbb_binarization (based on selectional auto-encoders). - - For each page of the input file group, open and deserialize input PAGE-XML - and its respective images. Then iterate over the element hierarchy down to - the requested ``operation_level``. - - For each segment element, retrieve a raw (non-binarized) segment image - according to the layout annotation (from an existing ``AlternativeImage``, - or by cropping into the higher-level images, and deskewing when applicable). - - Pass the image to the binarizer (which runs in fixed-size windows/patches - across the image and stitches the results together). - - Serialize the resulting bilevel image as PNG file and add it to the output - file group (with file ID suffix ``.IMG-BIN``) along with the output PAGE-XML - (referencing it as new ``AlternativeImage`` for the segment element). - - Produce a new PAGE output file by serialising the resulting hierarchy. - """ - assert input_pcgts - assert input_pcgts[0] - assert self.parameter - oplevel = self.parameter['operation_level'] - pcgts = input_pcgts[0] - result = OcrdPageResult(pcgts) - page = pcgts.get_Page() - page_image, page_xywh, _ = self.workspace.image_from_page( - page, page_id, feature_filter='binarized') - - if oplevel == 'page': - self.logger.info("Binarizing on 'page' level in page '%s'", page_id) - page_image_bin = cv2pil(self.binarizer.run(image=pil2cv(page_image), use_patches=True)) - # update PAGE (reference the image file): - page_image_ref = AlternativeImageType(comments=page_xywh['features'] + ',binarized,clipped') - page.add_AlternativeImage(page_image_ref) - result.images.append(OcrdPageResultImage(page_image_bin, '.IMG-BIN', page_image_ref)) - - elif oplevel == 'region': - regions = page.get_AllRegions(['Text', 'Table'], depth=1) - if not regions: - self.logger.warning("Page '%s' contains no text/table regions", page_id) - for region in regions: - region_image, region_xywh = self.workspace.image_from_segment( - region, page_image, page_xywh, feature_filter='binarized') - region_image_bin = cv2pil(self.binarizer.run(image=pil2cv(region_image), use_patches=True)) - # update PAGE (reference the image file): - region_image_ref = AlternativeImageType(comments=region_xywh['features'] + ',binarized') - region.add_AlternativeImage(region_image_ref) - result.images.append(OcrdPageResultImage(region_image_bin, region.id + '.IMG-BIN', region_image_ref)) - - elif oplevel == 'line': - lines = page.get_AllTextLines() - if not lines: - self.logger.warning("Page '%s' contains no text lines", page_id) - for line in lines: - line_image, line_xywh = self.workspace.image_from_segment(line, page_image, page_xywh, feature_filter='binarized') - line_image_bin = cv2pil(self.binarizer.run(image=pil2cv(line_image), use_patches=True)) - # update PAGE (reference the image file): - line_image_ref = AlternativeImageType(comments=line_xywh['features'] + ',binarized') - line.add_AlternativeImage(region_image_ref) - result.images.append(OcrdPageResultImage(line_image_bin, line.id + '.IMG-BIN', line_image_ref)) - - return result - -@command() -@ocrd_cli_options -def main(*args, **kwargs): - return ocrd_cli_wrap_processor(SbbBinarizeProcessor, *args, **kwargs) diff --git a/src/eynollah/processor.py b/src/eynollah/processor.py deleted file mode 100644 index 8f99489..0000000 --- a/src/eynollah/processor.py +++ /dev/null @@ -1,91 +0,0 @@ -from typing import Optional -from ocrd_models import OcrdPage -from ocrd import Processor, OcrdPageResult - -from .eynollah import Eynollah, EynollahXmlWriter - -class EynollahProcessor(Processor): - # already employs background CPU multiprocessing per page - # already employs GPU (without singleton process atm) - max_workers = 1 - - @property - def executable(self): - return 'ocrd-eynollah-segment' - - def setup(self) -> None: - if self.parameter['textline_light'] and not self.parameter['light_version']: - raise ValueError("Error: You set parameter 'textline_light' to enable light textline detection, " - "but parameter 'light_version' is not enabled") - self.eynollah = Eynollah( - self.resolve_resource(self.parameter['models']), - logger=self.logger, - allow_enhancement=self.parameter['allow_enhancement'], - curved_line=self.parameter['curved_line'], - right2left=self.parameter['right_to_left'], - ignore_page_extraction=self.parameter['ignore_page_extraction'], - light_version=self.parameter['light_version'], - textline_light=self.parameter['textline_light'], - full_layout=self.parameter['full_layout'], - allow_scaling=self.parameter['allow_scaling'], - headers_off=self.parameter['headers_off'], - tables=self.parameter['tables'], - ) - self.eynollah.plotter = None - - def shutdown(self): - if hasattr(self, 'eynollah'): - del self.eynollah - - def process_page_pcgts(self, *input_pcgts: Optional[OcrdPage], page_id: Optional[str] = None) -> OcrdPageResult: - """ - Performs cropping, region and line segmentation with Eynollah. - - For each page, open and deserialize PAGE input file (from existing - PAGE file in the input fileGrp, or generated from image file). - Retrieve its respective page-level image (ignoring annotation that - already added `binarized`, `cropped` or `deskewed` features). - - Set up Eynollah to detect regions and lines, and add each one to the - page, respectively. - - \b - - If ``tables``, try to detect table blocks and add them as TableRegion. - - If ``full_layout``, then in addition to paragraphs and marginals, also - try to detect drop capitals and headings. - - If ``ignore_page_extraction``, then attempt no cropping of the page. - - If ``curved_line``, then compute contour polygons for text lines - instead of simple bounding boxes. - - Produce a new output file by serialising the resulting hierarchy. - """ - assert input_pcgts - assert input_pcgts[0] - assert self.parameter - pcgts = input_pcgts[0] - result = OcrdPageResult(pcgts) - page = pcgts.get_Page() - page_image, _, _ = self.workspace.image_from_page( - page, page_id, - # avoid any features that would change the coordinate system: cropped,deskewed - # (the PAGE builder merely adds regions, so afterwards we would not know which to transform) - # also avoid binarization as models usually fare better on grayscale/RGB - feature_filter='cropped,deskewed,binarized') - if hasattr(page_image, 'filename'): - image_filename = page_image.filename - else: - image_filename = "dummy" # will be replaced by ocrd.Processor.process_page_file - result.images.append(OcrdPageResultImage(page_image, '.IMG', page)) # mark as new original - # FIXME: mask out already existing regions (incremental segmentation) - self.eynollah.cache_images( - image_pil=page_image, - dpi=self.parameter['dpi'], - ) - self.eynollah.writer = EynollahXmlWriter( - dir_out=None, - image_filename=image_filename, - curved_line=self.eynollah.curved_line, - textline_light=self.eynollah.textline_light, - pcgts=pcgts) - self.eynollah.run_single() - return result diff --git a/src/eynollah/sbb_binarize.py b/src/eynollah/sbb_binarize.py deleted file mode 100644 index f43b6ba..0000000 --- a/src/eynollah/sbb_binarize.py +++ /dev/null @@ -1,377 +0,0 @@ -""" -Tool to load model and binarize a given image. -""" - -import sys -from glob import glob -import os -import logging - -import numpy as np -from PIL import Image -import cv2 -from ocrd_utils import tf_disable_interactive_logs -tf_disable_interactive_logs() -import tensorflow as tf -from tensorflow.keras.models import load_model -from tensorflow.python.keras import backend as tensorflow_backend - - -def resize_image(img_in, input_height, input_width): - return cv2.resize(img_in, (input_width, input_height), interpolation=cv2.INTER_NEAREST) - -class SbbBinarizer: - - def __init__(self, model_dir, logger=None): - self.model_dir = model_dir - self.log = logger if logger else logging.getLogger('SbbBinarizer') - - self.start_new_session() - - self.model_files = glob(self.model_dir+"/*/", recursive = True) - - self.models = [] - for model_file in self.model_files: - self.models.append(self.load_model(model_file)) - - def start_new_session(self): - config = tf.compat.v1.ConfigProto() - config.gpu_options.allow_growth = True - - self.session = tf.compat.v1.Session(config=config) # tf.InteractiveSession() - tensorflow_backend.set_session(self.session) - - def end_session(self): - tensorflow_backend.clear_session() - self.session.close() - del self.session - - def load_model(self, model_name): - model = load_model(os.path.join(self.model_dir, model_name), compile=False) - model_height = model.layers[len(model.layers)-1].output_shape[1] - model_width = model.layers[len(model.layers)-1].output_shape[2] - n_classes = model.layers[len(model.layers)-1].output_shape[3] - return model, model_height, model_width, n_classes - - def predict(self, model_in, img, use_patches, n_batch_inference=5): - tensorflow_backend.set_session(self.session) - model, model_height, model_width, n_classes = model_in - - img_org_h = img.shape[0] - img_org_w = img.shape[1] - - if img.shape[0] < model_height and img.shape[1] >= model_width: - img_padded = np.zeros(( model_height, img.shape[1], img.shape[2] )) - - index_start_h = int( abs( img.shape[0] - model_height) /2.) - index_start_w = 0 - - img_padded [ index_start_h: index_start_h+img.shape[0], :, : ] = img[:,:,:] - - elif img.shape[0] >= model_height and img.shape[1] < model_width: - img_padded = np.zeros(( img.shape[0], model_width, img.shape[2] )) - - index_start_h = 0 - index_start_w = int( abs( img.shape[1] - model_width) /2.) - - img_padded [ :, index_start_w: index_start_w+img.shape[1], : ] = img[:,:,:] - - - elif img.shape[0] < model_height and img.shape[1] < model_width: - img_padded = np.zeros(( model_height, model_width, img.shape[2] )) - - index_start_h = int( abs( img.shape[0] - model_height) /2.) - index_start_w = int( abs( img.shape[1] - model_width) /2.) - - img_padded [ index_start_h: index_start_h+img.shape[0], index_start_w: index_start_w+img.shape[1], : ] = img[:,:,:] - - else: - index_start_h = 0 - index_start_w = 0 - img_padded = np.copy(img) - - - img = np.copy(img_padded) - - - - if use_patches: - - margin = int(0.1 * model_width) - - width_mid = model_width - 2 * margin - height_mid = model_height - 2 * margin - - - img = img / float(255.0) - - img_h = img.shape[0] - img_w = img.shape[1] - - prediction_true = np.zeros((img_h, img_w, 3)) - mask_true = np.zeros((img_h, img_w)) - nxf = img_w / float(width_mid) - nyf = img_h / float(height_mid) - - if nxf > int(nxf): - nxf = int(nxf) + 1 - else: - nxf = int(nxf) - - if nyf > int(nyf): - nyf = int(nyf) + 1 - else: - nyf = int(nyf) - - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - - img_patch = np.zeros((n_batch_inference, model_height, model_width,3)) - - for i in range(nxf): - for j in range(nyf): - - if i == 0: - index_x_d = i * width_mid - index_x_u = index_x_d + model_width - elif i > 0: - index_x_d = i * width_mid - index_x_u = index_x_d + model_width - - if j == 0: - index_y_d = j * height_mid - index_y_u = index_y_d + model_height - elif j > 0: - index_y_d = j * height_mid - index_y_u = index_y_d + model_height - - if index_x_u > img_w: - index_x_u = img_w - index_x_d = img_w - model_width - if index_y_u > img_h: - index_y_u = img_h - index_y_d = img_h - model_height - - - list_i_s.append(i) - list_j_s.append(j) - list_x_u.append(index_x_u) - list_x_d.append(index_x_d) - list_y_d.append(index_y_d) - list_y_u.append(index_y_u) - - - img_patch[batch_indexer,:,:,:] = img[index_y_d:index_y_u, index_x_d:index_x_u, :] - - batch_indexer = batch_indexer + 1 - - - - if batch_indexer == n_batch_inference: - - label_p_pred = model.predict(img_patch,verbose=0) - - seg = np.argmax(label_p_pred, axis=3) - - #print(seg.shape, len(seg), len(list_i_s)) - - indexer_inside_batch = 0 - for i_batch, j_batch in zip(list_i_s, list_j_s): - seg_in = seg[indexer_inside_batch,:,:] - seg_color = np.repeat(seg_in[:, :, np.newaxis], 3, axis=2) - - index_y_u_in = list_y_u[indexer_inside_batch] - index_y_d_in = list_y_d[indexer_inside_batch] - - index_x_u_in = list_x_u[indexer_inside_batch] - index_x_d_in = list_x_d[indexer_inside_batch] - - if i_batch == 0 and j_batch == 0: - seg_color = seg_color[0 : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + 0 : index_y_u_in - margin, index_x_d_in + 0 : index_x_u_in - margin, :] = seg_color - elif i_batch == nxf - 1 and j_batch == nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - 0, :] - prediction_true[index_y_d_in + margin : index_y_u_in - 0, index_x_d_in + margin : index_x_u_in - 0, :] = seg_color - elif i_batch == 0 and j_batch == nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - 0, 0 : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - 0, index_x_d_in + 0 : index_x_u_in - margin, :] = seg_color - elif i_batch == nxf - 1 and j_batch == 0: - seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] - prediction_true[index_y_d_in + 0 : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - 0, :] = seg_color - elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - margin, index_x_d_in + 0 : index_x_u_in - margin, :] = seg_color - elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] - prediction_true[index_y_d_in + margin : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - 0, :] = seg_color - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: - seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + 0 : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - margin, :] = seg_color - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - 0, index_x_d_in + margin : index_x_u_in - margin, :] = seg_color - else: - seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - margin, :] = seg_color - - indexer_inside_batch = indexer_inside_batch +1 - - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - - img_patch = np.zeros((n_batch_inference, model_height, model_width,3)) - - elif i==(nxf-1) and j==(nyf-1): - label_p_pred = model.predict(img_patch,verbose=0) - - seg = np.argmax(label_p_pred, axis=3) - - #print(seg.shape, len(seg), len(list_i_s)) - - indexer_inside_batch = 0 - for i_batch, j_batch in zip(list_i_s, list_j_s): - seg_in = seg[indexer_inside_batch,:,:] - seg_color = np.repeat(seg_in[:, :, np.newaxis], 3, axis=2) - - index_y_u_in = list_y_u[indexer_inside_batch] - index_y_d_in = list_y_d[indexer_inside_batch] - - index_x_u_in = list_x_u[indexer_inside_batch] - index_x_d_in = list_x_d[indexer_inside_batch] - - if i_batch == 0 and j_batch == 0: - seg_color = seg_color[0 : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + 0 : index_y_u_in - margin, index_x_d_in + 0 : index_x_u_in - margin, :] = seg_color - elif i_batch == nxf - 1 and j_batch == nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - 0, :] - prediction_true[index_y_d_in + margin : index_y_u_in - 0, index_x_d_in + margin : index_x_u_in - 0, :] = seg_color - elif i_batch == 0 and j_batch == nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - 0, 0 : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - 0, index_x_d_in + 0 : index_x_u_in - margin, :] = seg_color - elif i_batch == nxf - 1 and j_batch == 0: - seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] - prediction_true[index_y_d_in + 0 : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - 0, :] = seg_color - elif i_batch == 0 and j_batch != 0 and j_batch != nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - margin, 0 : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - margin, index_x_d_in + 0 : index_x_u_in - margin, :] = seg_color - elif i_batch == nxf - 1 and j_batch != 0 and j_batch != nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - 0, :] - prediction_true[index_y_d_in + margin : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - 0, :] = seg_color - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == 0: - seg_color = seg_color[0 : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + 0 : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - margin, :] = seg_color - elif i_batch != 0 and i_batch != nxf - 1 and j_batch == nyf - 1: - seg_color = seg_color[margin : seg_color.shape[0] - 0, margin : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - 0, index_x_d_in + margin : index_x_u_in - margin, :] = seg_color - else: - seg_color = seg_color[margin : seg_color.shape[0] - margin, margin : seg_color.shape[1] - margin, :] - prediction_true[index_y_d_in + margin : index_y_u_in - margin, index_x_d_in + margin : index_x_u_in - margin, :] = seg_color - - indexer_inside_batch = indexer_inside_batch +1 - - - list_i_s = [] - list_j_s = [] - list_x_u = [] - list_x_d = [] - list_y_u = [] - list_y_d = [] - - batch_indexer = 0 - - img_patch = np.zeros((n_batch_inference, model_height, model_width,3)) - - - - prediction_true = prediction_true[index_start_h: index_start_h+img_org_h, index_start_w: index_start_w+img_org_w,:] - prediction_true = prediction_true.astype(np.uint8) - - else: - img_h_page = img.shape[0] - img_w_page = img.shape[1] - img = img / float(255.0) - img = resize_image(img, model_height, model_width) - - label_p_pred = model.predict(img.reshape(1, img.shape[0], img.shape[1], img.shape[2])) - - seg = np.argmax(label_p_pred, axis=3)[0] - seg_color = np.repeat(seg[:, :, np.newaxis], 3, axis=2) - prediction_true = resize_image(seg_color, img_h_page, img_w_page) - prediction_true = prediction_true.astype(np.uint8) - return prediction_true[:,:,0] - - def run(self, image=None, image_path=None, save=None, use_patches=False, dir_in=None, dir_out=None): - print(dir_in,'dir_in') - if not dir_in: - if (image is not None and image_path is not None) or \ - (image is None and image_path is None): - raise ValueError("Must pass either a opencv2 image or an image_path") - if image_path is not None: - image = cv2.imread(image_path) - img_last = 0 - for n, (model, model_file) in enumerate(zip(self.models, self.model_files)): - self.log.info('Predicting with model %s [%s/%s]' % (model_file, n + 1, len(self.model_files))) - - res = self.predict(model, image, use_patches) - - img_fin = np.zeros((res.shape[0], res.shape[1], 3)) - res[:, :][res[:, :] == 0] = 2 - res = res - 1 - res = res * 255 - img_fin[:, :, 0] = res - img_fin[:, :, 1] = res - img_fin[:, :, 2] = res - - img_fin = img_fin.astype(np.uint8) - img_fin = (res[:, :] == 0) * 255 - img_last = img_last + img_fin - - kernel = np.ones((5, 5), np.uint8) - img_last[:, :][img_last[:, :] > 0] = 255 - img_last = (img_last[:, :] == 0) * 255 - if save: - cv2.imwrite(save, img_last) - return img_last - else: - ls_imgs = os.listdir(dir_in) - for image_name in ls_imgs: - image_stem = image_name.split('.')[0] - print(image_name,'image_name') - image = cv2.imread(os.path.join(dir_in,image_name) ) - img_last = 0 - for n, (model, model_file) in enumerate(zip(self.models, self.model_files)): - self.log.info('Predicting with model %s [%s/%s]' % (model_file, n + 1, len(self.model_files))) - - res = self.predict(model, image, use_patches) - - img_fin = np.zeros((res.shape[0], res.shape[1], 3)) - res[:, :][res[:, :] == 0] = 2 - res = res - 1 - res = res * 255 - img_fin[:, :, 0] = res - img_fin[:, :, 1] = res - img_fin[:, :, 2] = res - - img_fin = img_fin.astype(np.uint8) - img_fin = (res[:, :] == 0) * 255 - img_last = img_last + img_fin - - kernel = np.ones((5, 5), np.uint8) - img_last[:, :][img_last[:, :] > 0] = 255 - img_last = (img_last[:, :] == 0) * 255 - - cv2.imwrite(os.path.join(dir_out,image_stem+'.png'), img_last) diff --git a/src/eynollah/utils/contour.py b/src/eynollah/utils/contour.py deleted file mode 100644 index 0e84153..0000000 --- a/src/eynollah/utils/contour.py +++ /dev/null @@ -1,334 +0,0 @@ -from functools import partial -import cv2 -import numpy as np -from shapely import geometry - -from .rotate import rotate_image, rotation_image_new - -def contours_in_same_horizon(cy_main_hor): - X1 = np.zeros((len(cy_main_hor), len(cy_main_hor))) - X2 = np.zeros((len(cy_main_hor), len(cy_main_hor))) - - X1[0::1, :] = cy_main_hor[:] - X2 = X1.T - - X_dif = np.abs(X2 - X1) - args_help = np.array(range(len(cy_main_hor))) - all_args = [] - for i in range(len(cy_main_hor)): - list_h = list(args_help[X_dif[i, :] <= 20]) - list_h.append(i) - if len(list_h) > 1: - all_args.append(list(set(list_h))) - return np.unique(np.array(all_args, dtype=object)) - -def find_contours_mean_y_diff(contours_main): - M_main = [cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) for j in range(len(M_main))] - return np.mean(np.diff(np.sort(np.array(cy_main)))) - -def get_text_region_boxes_by_given_contours(contours): - boxes = [] - contours_new = [] - for jj in range(len(contours)): - box = cv2.boundingRect(contours[jj]) - boxes.append(box) - contours_new.append(contours[jj]) - - return boxes, contours_new - -def filter_contours_area_of_image(image, contours, hierarchy, max_area, min_area): - found_polygons_early = [] - for jv,c in enumerate(contours): - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - area = polygon.area - if (area >= min_area * np.prod(image.shape[:2]) and - area <= max_area * np.prod(image.shape[:2]) and - hierarchy[0][jv][3] == -1): - found_polygons_early.append(np.array([[point] - for point in polygon.exterior.coords], dtype=np.uint)) - return found_polygons_early - -def filter_contours_area_of_image_tables(image, contours, hierarchy, max_area, min_area): - found_polygons_early = [] - for jv,c in enumerate(contours): - if len(c) < 3: # A polygon cannot have less than 3 points - continue - - polygon = geometry.Polygon([point[0] for point in c]) - # area = cv2.contourArea(c) - area = polygon.area - ##print(np.prod(thresh.shape[:2])) - # Check that polygon has area greater than minimal area - # print(hierarchy[0][jv][3],hierarchy ) - if (area >= min_area * np.prod(image.shape[:2]) and - area <= max_area * np.prod(image.shape[:2]) and - # hierarchy[0][jv][3]==-1 - True): - # print(c[0][0][1]) - found_polygons_early.append(np.array([[point] - for point in polygon.exterior.coords], dtype=np.int32)) - return found_polygons_early - -def find_new_features_of_contours(contours_main): - areas_main = np.array([cv2.contourArea(contours_main[j]) - for j in range(len(contours_main))]) - M_main = [cv2.moments(contours_main[j]) - for j in range(len(contours_main))] - cx_main = [(M_main[j]["m10"] / (M_main[j]["m00"] + 1e-32)) - for j in range(len(M_main))] - cy_main = [(M_main[j]["m01"] / (M_main[j]["m00"] + 1e-32)) - for j in range(len(M_main))] - try: - x_min_main = np.array([np.min(contours_main[j][:, 0, 0]) - for j in range(len(contours_main))]) - argmin_x_main = np.array([np.argmin(contours_main[j][:, 0, 0]) - for j in range(len(contours_main))]) - x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 0] - for j in range(len(contours_main))]) - y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0, 1] - for j in range(len(contours_main))]) - x_max_main = np.array([np.max(contours_main[j][:, 0, 0]) - for j in range(len(contours_main))]) - y_min_main = np.array([np.min(contours_main[j][:, 0, 1]) - for j in range(len(contours_main))]) - y_max_main = np.array([np.max(contours_main[j][:, 0, 1]) - for j in range(len(contours_main))]) - except: - x_min_main = np.array([np.min(contours_main[j][:, 0]) - for j in range(len(contours_main))]) - argmin_x_main = np.array([np.argmin(contours_main[j][:, 0]) - for j in range(len(contours_main))]) - x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 0] - for j in range(len(contours_main))]) - y_corr_x_min_from_argmin = np.array([contours_main[j][argmin_x_main[j], 1] - for j in range(len(contours_main))]) - x_max_main = np.array([np.max(contours_main[j][:, 0]) - for j in range(len(contours_main))]) - y_min_main = np.array([np.min(contours_main[j][:, 1]) - for j in range(len(contours_main))]) - y_max_main = np.array([np.max(contours_main[j][:, 1]) - for j in range(len(contours_main))]) - # dis_x=np.abs(x_max_main-x_min_main) - - return cx_main, cy_main, x_min_main, x_max_main, y_min_main, y_max_main, y_corr_x_min_from_argmin - -def find_features_of_contours(contours_main): - areas_main=np.array([cv2.contourArea(contours_main[j]) for j in range(len(contours_main))]) - M_main=[cv2.moments(contours_main[j]) for j in range(len(contours_main))] - cx_main=[(M_main[j]['m10']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - cy_main=[(M_main[j]['m01']/(M_main[j]['m00']+1e-32)) for j in range(len(M_main))] - x_min_main=np.array([np.min(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - x_max_main=np.array([np.max(contours_main[j][:,0,0]) for j in range(len(contours_main))]) - - y_min_main=np.array([np.min(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - y_max_main=np.array([np.max(contours_main[j][:,0,1]) for j in range(len(contours_main))]) - - return y_min_main, y_max_main - -def return_parent_contours(contours, hierarchy): - contours_parent = [contours[i] - for i in range(len(contours)) - if hierarchy[0][i][3] == -1] - return contours_parent - -def return_contours_of_interested_region(region_pre_p, pixel, min_area=0.0002): - # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 - else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables(thresh, contours_imgs, hierarchy, - max_area=1, min_area=min_area) - return contours_imgs - -def do_work_of_contours_in_image(contour, index_r_con, img, slope_first): - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[contour], color=(1, 1, 1)) - - img_copy = rotation_image_new(img_copy, -slope_first) - img_copy = img_copy.astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) - cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) - - return cont_int[0], index_r_con - -def get_textregion_contours_in_org_image_multi(cnts, img, slope_first, map=map): - if not len(cnts): - return [], [] - results = map(partial(do_work_of_contours_in_image, - img=img, - slope_first=slope_first, - ), - cnts, range(len(cnts))) - return tuple(zip(*results)) - -def get_textregion_contours_in_org_image(cnts, img, slope_first): - cnts_org = [] - # print(cnts,'cnts') - for i in range(len(cnts)): - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[cnts[i]], color=(1, 1, 1)) - - # plt.imshow(img_copy) - # plt.show() - - # print(img.shape,'img') - img_copy = rotation_image_new(img_copy, -slope_first) - ##print(img_copy.shape,'img_copy') - # plt.imshow(img_copy) - # plt.show() - - img_copy = img_copy.astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) - cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) - # print(np.shape(cont_int[0])) - cnts_org.append(cont_int[0]) - - return cnts_org - -def get_textregion_contours_in_org_image_light_old(cnts, img, slope_first): - zoom = 3 - img = cv2.resize(img, (img.shape[1] // zoom, - img.shape[0] // zoom), - interpolation=cv2.INTER_NEAREST) - cnts_org = [] - for cnt in cnts: - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[(cnt / zoom).astype(int)], color=(1, 1, 1)) - - img_copy = rotation_image_new(img_copy, -slope_first).astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) - cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) - cnts_org.append(cont_int[0] * zoom) - - return cnts_org - -def do_back_rotation_and_get_cnt_back(contour_par, index_r_con, img, slope_first, confidence_matrix): - img_copy = np.zeros(img.shape) - img_copy = cv2.fillPoly(img_copy, pts=[contour_par], color=(1, 1, 1)) - confidence_matrix_mapped_with_contour = confidence_matrix * img_copy[:,:,0] - confidence_contour = np.sum(confidence_matrix_mapped_with_contour) / float(np.sum(img_copy[:,:,0])) - - img_copy = rotation_image_new(img_copy, -slope_first).astype(np.uint8) - imgray = cv2.cvtColor(img_copy, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - cont_int, _ = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - if len(cont_int)==0: - cont_int = [] - cont_int.append(contour_par) - confidence_contour = 0 - else: - cont_int[0][:, 0, 0] = cont_int[0][:, 0, 0] + np.abs(img_copy.shape[1] - img.shape[1]) - cont_int[0][:, 0, 1] = cont_int[0][:, 0, 1] + np.abs(img_copy.shape[0] - img.shape[0]) - return cont_int[0], index_r_con, confidence_contour - -def get_textregion_contours_in_org_image_light(cnts, img, slope_first, confidence_matrix, map=map): - if not len(cnts): - return [], [] - - confidence_matrix = cv2.resize(confidence_matrix, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) - img = cv2.resize(img, (int(img.shape[1]/6), int(img.shape[0]/6)), interpolation=cv2.INTER_NEAREST) - ##cnts = list( (np.array(cnts)/2).astype(np.int16) ) - #cnts = cnts/2 - cnts = [(i/6).astype(int) for i in cnts] - results = map(partial(do_back_rotation_and_get_cnt_back, - img=img, - slope_first=slope_first, - confidence_matrix=confidence_matrix, - ), - cnts, range(len(cnts))) - contours, indexes, conf_contours = tuple(zip(*results)) - return [i*6 for i in contours], list(conf_contours) - -def return_contours_of_interested_textline(region_pre_p, pixel): - # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 - else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables( - thresh, contours_imgs, hierarchy, max_area=1, min_area=0.000000003) - return contours_imgs - -def return_contours_of_image(image): - if len(image.shape) == 2: - image = np.repeat(image[:, :, np.newaxis], 3, axis=2) - image = image.astype(np.uint8) - else: - image = image.astype(np.uint8) - imgray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - return contours, hierarchy - -def return_contours_of_interested_region_by_min_size(region_pre_p, pixel, min_size=0.00003): - # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 - else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables( - thresh, contours_imgs, hierarchy, max_area=1, min_area=min_size) - - return contours_imgs - -def return_contours_of_interested_region_by_size(region_pre_p, pixel, min_area, max_area): - # pixels of images are identified by 5 - if len(region_pre_p.shape) == 3: - cnts_images = (region_pre_p[:, :, 0] == pixel) * 1 - else: - cnts_images = (region_pre_p[:, :] == pixel) * 1 - cnts_images = cnts_images.astype(np.uint8) - cnts_images = np.repeat(cnts_images[:, :, np.newaxis], 3, axis=2) - imgray = cv2.cvtColor(cnts_images, cv2.COLOR_BGR2GRAY) - ret, thresh = cv2.threshold(imgray, 0, 255, 0) - contours_imgs, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - - contours_imgs = return_parent_contours(contours_imgs, hierarchy) - contours_imgs = filter_contours_area_of_image_tables( - thresh, contours_imgs, hierarchy, max_area=max_area, min_area=min_area) - - img_ret = np.zeros((region_pre_p.shape[0], region_pre_p.shape[1], 3)) - img_ret = cv2.fillPoly(img_ret, pts=contours_imgs, color=(1, 1, 1)) - - return img_ret[:, :, 0] - diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 0000000..9de35ef --- /dev/null +++ b/tests/base.py @@ -0,0 +1,54 @@ +# pylint: disable=unused-import + +from os.path import dirname, realpath +from os import chdir +import sys +import logging +import io +import collections +from unittest import TestCase as VanillaTestCase, skip, main as unittests_main +import pytest +from ocrd_utils import disableLogging, initLogging + +def main(fn=None): + if fn: + sys.exit(pytest.main([fn])) + else: + unittests_main() + +class TestCase(VanillaTestCase): + + @classmethod + def setUpClass(cls): + chdir(dirname(realpath(__file__)) + '/..') + + def setUp(self): + disableLogging() + initLogging() + +class CapturingTestCase(TestCase): + """ + A TestCase that needs to capture stderr/stdout and invoke click CLI. + """ + + @pytest.fixture(autouse=True) + def _setup_pytest_capfd(self, capfd): + self.capfd = capfd + + def invoke_cli(self, cli, args): + """ + Substitution for click.CliRunner.invooke that works together nicely + with unittests/pytest capturing stdout/stderr. + """ + self.capture_out_err() # XXX snapshot just before executing the CLI + code = 0 + sys.argv[1:] = args # XXX necessary because sys.argv reflects pytest args not cli args + try: + cli.main(args=args) + except SystemExit as e: + code = e.code + out, err = self.capture_out_err() + return code, out, err + + def capture_out_err(self): + return self.capfd.readouterr() diff --git a/tests/resources/euler_rechenkunst01_1738_0025.tif b/tests/resources/euler_rechenkunst01_1738_0025.tif deleted file mode 100644 index db6bae1..0000000 Binary files a/tests/resources/euler_rechenkunst01_1738_0025.tif and /dev/null differ diff --git a/tests/test_counter.py b/tests/test_counter.py index 4f1abe2..8ef0756 100644 --- a/tests/test_counter.py +++ b/tests/test_counter.py @@ -1,4 +1,5 @@ -from eynollah.utils.counter import EynollahIdCounter +from tests.base import main +from qurator.eynollah.utils.counter import EynollahIdCounter def test_counter_string(): c = EynollahIdCounter() @@ -28,3 +29,5 @@ def test_counter_methods(): c.inc('region', -9) assert c.get('region') == 1 +if __name__ == '__main__': + main(__file__) diff --git a/tests/test_dpi.py b/tests/test_dpi.py index abe48ce..510ffc5 100644 --- a/tests/test_dpi.py +++ b/tests/test_dpi.py @@ -1,8 +1,11 @@ import cv2 from pathlib import Path -from eynollah.utils.pil_cv2 import check_dpi +from qurator.eynollah.utils.pil_cv2 import check_dpi +from tests.base import main def test_dpi(): fpath = str(Path(__file__).parent.joinpath('resources', 'kant_aufklaerung_1784_0020.tif')) assert 230 == check_dpi(cv2.imread(fpath)) +if __name__ == '__main__': + main(__file__) diff --git a/tests/test_run.py b/tests/test_run.py index 607140e..b1137e7 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,132 +1,24 @@ from os import environ from pathlib import Path -import logging -from PIL import Image -from eynollah.cli import layout as layout_cli, binarization as binarization_cli -from click.testing import CliRunner -from ocrd_modelfactory import page_from_file -from ocrd_models.constants import NAMESPACES as NS +from ocrd_utils import pushd_popd +from tests.base import CapturingTestCase as TestCase, main +from qurator.eynollah.cli import main as eynollah_cli testdir = Path(__file__).parent.resolve() EYNOLLAH_MODELS = environ.get('EYNOLLAH_MODELS', str(testdir.joinpath('..', 'models_eynollah').resolve())) -SBBBIN_MODELS = environ.get('SBBBIN_MODELS', str(testdir.joinpath('..', 'default-2021-03-09').resolve())) -def test_run_eynollah_layout_filename(tmp_path, subtests, pytestconfig, caplog): - infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') - outfile = tmp_path / 'kant_aufklaerung_1784_0020.xml' - args = [ - '-m', EYNOLLAH_MODELS, - '-i', str(infile), - '-o', str(outfile.parent), - # subtests write to same location - '--overwrite', - ] - if pytestconfig.getoption('verbose') > 0: - args.extend(['-l', 'DEBUG']) - caplog.set_level(logging.INFO) - def only_eynollah(logrec): - return logrec.name == 'eynollah' - runner = CliRunner() - for options in [ - [], # defaults - ["--allow_scaling", "--curved-line"], - ["--allow_scaling", "--curved-line", "--full-layout"], - ["--allow_scaling", "--curved-line", "--full-layout", "--reading_order_machine_based"], - ["--allow_scaling", "--curved-line", "--full-layout", "--reading_order_machine_based", - "--textline_light", "--light_version"], - # -ep ... - # -eoi ... - # --do_ocr - # --skip_layout_and_reading_order - ]: - with subtests.test(#msg="test CLI", - options=options): - with caplog.filtering(only_eynollah): - result = runner.invoke(layout_cli, args + options, catch_exceptions=False) - print(result) - assert result.exit_code == 0 - logmsgs = [logrec.message for logrec in caplog.records] - assert str(infile) in logmsgs - assert outfile.exists() - tree = page_from_file(str(outfile)).etree - regions = tree.xpath("//page:TextRegion", namespaces=NS) - assert len(regions) >= 2, "result is inaccurate" - regions = tree.xpath("//page:SeparatorRegion", namespaces=NS) - assert len(regions) >= 2, "result is inaccurate" - lines = tree.xpath("//page:TextLine", namespaces=NS) - assert len(lines) == 31, "result is inaccurate" # 29 paragraph lines, 1 page and 1 catch-word line +class TestEynollahRun(TestCase): -def test_run_eynollah_layout_directory(tmp_path, pytestconfig, caplog): - indir = testdir.joinpath('resources') - outdir = tmp_path - args = [ - '-m', EYNOLLAH_MODELS, - '-di', str(indir), - '-o', str(outdir), - ] - if pytestconfig.getoption('verbose') > 0: - args.extend(['-l', 'DEBUG']) - caplog.set_level(logging.INFO) - def only_eynollah(logrec): - return logrec.name == 'eynollah' - runner = CliRunner() - with caplog.filtering(only_eynollah): - result = runner.invoke(layout_cli, args) - print(result) - assert result.exit_code == 0 - logmsgs = [logrec.message for logrec in caplog.records] - assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Job done in')]) == 2 - assert any(logmsg for logmsg in logmsgs if logmsg.startswith('All jobs done in')) - assert len(list(outdir.iterdir())) == 2 + def test_full_run(self): + with pushd_popd(tempdir=True) as tempdir: + code, out, err = self.invoke_cli(eynollah_cli, [ + '-m', EYNOLLAH_MODELS, + '-i', str(testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif')), + '-o', tempdir + ]) + print(code, out, err) + assert not code -def test_run_eynollah_binarization_filename(tmp_path, subtests, pytestconfig, caplog): - infile = testdir.joinpath('resources/kant_aufklaerung_1784_0020.tif') - outfile = tmp_path.joinpath('kant_aufklaerung_1784_0020.png') - args = [ - '-m', SBBBIN_MODELS, - str(infile), - str(outfile), - ] - caplog.set_level(logging.INFO) - def only_eynollah(logrec): - return logrec.name == 'SbbBinarizer' - runner = CliRunner() - for options in [ - [], # defaults - ["--no-patches"], - ]: - with subtests.test(#msg="test CLI", - options=options): - with caplog.filtering(only_eynollah): - result = runner.invoke(binarization_cli, args + options) - print(result) - assert result.exit_code == 0 - logmsgs = [logrec.message for logrec in caplog.records] - assert any(True for logmsg in logmsgs if logmsg.startswith('Predicting')) - assert outfile.exists() - with Image.open(infile) as original_img: - original_size = original_img.size - with Image.open(outfile) as binarized_img: - binarized_size = binarized_img.size - assert original_size == binarized_size - -def test_run_eynollah_binarization_directory(tmp_path, subtests, pytestconfig, caplog): - indir = testdir.joinpath('resources') - outdir = tmp_path - args = [ - '-m', SBBBIN_MODELS, - '-di', str(indir), - '-do', str(outdir), - ] - caplog.set_level(logging.INFO) - def only_eynollah(logrec): - return logrec.name == 'SbbBinarizer' - runner = CliRunner() - with caplog.filtering(only_eynollah): - result = runner.invoke(binarization_cli, args) - print(result) - assert result.exit_code == 0 - logmsgs = [logrec.message for logrec in caplog.records] - assert len([logmsg for logmsg in logmsgs if logmsg.startswith('Predicting')]) == 2 - assert len(list(outdir.iterdir())) == 2 +if __name__ == '__main__': + main(__file__) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 252213f..d069479 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -1,7 +1,7 @@ def test_utils_import(): - import eynollah.utils - import eynollah.utils.contour - import eynollah.utils.drop_capitals - import eynollah.utils.drop_capitals - import eynollah.utils.is_nan - import eynollah.utils.rotate + import qurator.eynollah.utils + import qurator.eynollah.utils.contour + import qurator.eynollah.utils.drop_capitals + import qurator.eynollah.utils.drop_capitals + import qurator.eynollah.utils.is_nan + import qurator.eynollah.utils.rotate diff --git a/tests/test_xml.py b/tests/test_xml.py index 5dffc94..8422fd1 100644 --- a/tests/test_xml.py +++ b/tests/test_xml.py @@ -1,4 +1,5 @@ -from eynollah.utils.xml import create_page_xml +from pytest import main +from qurator.eynollah.utils.xml import create_page_xml from ocrd_models.ocrd_page import to_xml PAGE_2019 = 'http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15' @@ -8,3 +9,6 @@ def test_create_xml(): xmlstr = to_xml(pcgts) assert 'xmlns:pc="%s"' % PAGE_2019 in xmlstr assert 'Metadata' in xmlstr + +if __name__ == '__main__': + main([__file__])