Merge pull request #59 from bertsky/change-model-url

update to GH release archive as model URL
pull/65/head
Clemens Neudecker 2 years ago committed by GitHub
commit 7afe7574b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1,45 +1,31 @@
version: 2 version: 2.1
jobs: jobs:
build-python:
build-python37: parameters:
python-version:
type: string
docker: docker:
- image: python:3.7 - image: cimg/python:<< parameters.python-version >>
steps: steps:
- checkout - checkout
- restore_cache: - restore_cache:
keys: keys:
- ocrd-resources-{{ checksum "requirements.txt" }}-{{ checksum "Makefile" }} - ocrd-resources-{{ checksum "requirements.txt" }}-{{ checksum "Makefile" }}
- run: make install - run: make install
- run: make model - run: make models
- save_cache: - save_cache:
key: ocrd-resources-{{ checksum "requirements.txt" }}-{{ checksum "Makefile" }} key: ocrd-resources-{{ checksum "requirements.txt" }}-{{ checksum "Makefile" }}
paths: paths:
ocrd-resources ocrd-resources
- run: git submodule update --init
- run: make test - run: make test
build-python38:
docker:
- image: python:3.8
steps:
- checkout
- restore_cache:
keys:
- ocrd-resources-{{ checksum "requirements.txt" }}-{{ checksum "Makefile" }}
- run: make install
- run: make model
- save_cache:
key: ocrd-resources-{{ checksum "requirements.txt" }}-{{ checksum "Makefile" }}
paths:
ocrd-resources
- run: git submodule update --init
- run: make test
workflows: workflows:
version: 2
build: build:
jobs: jobs:
- build-python37 - build-python:
- build-python38 matrix:
parameters:
python-version: ['3.7', '3.8', '3.9', '3.10']

@ -0,0 +1,43 @@
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
name: CLI Tests
on: [push, pull_request]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10']
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v3
with:
python-version: ${{ matrix.python-version }}
- name: Lint with flake8
run: |
pip install flake8
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Install package
run: make install
- name: Cache models
uses: actions/cache@v3
with:
key: models
path: /home/runner/.local/share/ocrd-resources/ocrd-sbb-binarize/*
- name: Install dependencies for test
# also downloads models, if not already present
run: make models test/assets
- name: Run tests
run: make test
- name: Setup upterm session when failure
if: failure()
uses: lhotari/action-upterm@v1

@ -4,7 +4,19 @@ Versioned according to [Semantic Versioning](http://semver.org/).
## Unreleased ## Unreleased
## [0.0.10] - 2022-10-24 Fixed:
* repaired `operation_level=region` (typo)
* repaired standalone CLI `--version` (pkg name)
* repaired standalone CLI channel format (uint8)
Changed:
* Trained models loadable and registered in SavedModel format
* Test both models, deployed normally (not in CWD)
* Test input with actual regions in `operation_level=region`
## [0.0.11] - 2022-10-24
Added: Added:

@ -1,16 +1,17 @@
# BEGIN-EVAL makefile-parser --make-help Makefile # BEGIN-EVAL makefile-parser --make-help Makefile
.PHONY: help install
help: help:
@echo "" @echo ""
@echo " Targets" @echo " Targets"
@echo "" @echo ""
@echo " install Install with pip" @echo " install Install with pip"
@echo " model Downloads the pre-trained models from qurator-data.de" @echo " models Downloads the pre-trained models from qurator-data.de"
@echo " test Run tests" @echo " test Run tests"
@echo " clean Remove copies/results in test/assets"
@echo "" @echo ""
@echo " Variables" @echo " Variables"
@echo "" @echo ""
@echo " MODEL_DIR Directory to store models"
# END-EVAL # END-EVAL
@ -19,11 +20,26 @@ install:
pip install . pip install .
# Downloads the pre-trained models from qurator-data.de # Downloads the pre-trained models from qurator-data.de
.PHONY: model .PHONY: models
model: models:
ocrd resmgr download --allow-uninstalled --location cwd ocrd-sbb-binarize default ocrd resmgr download ocrd-sbb-binarize "*"
repo/assets/data:
git submodule update --init
# Setup test data
test/assets: repo/assets/data
@mkdir -p $@
cp -r -t $@ $</*
# Run tests # Run tests
test: model .PHONY: test
ocrd-sbb-binarize -m repo/assets/data/kant_aufklaerung_1784/data/mets.xml -I OCR-D-IMG -O BIN -P model default test: test/assets models
ocrd-sbb-binarize -m repo/assets/data/kant_aufklaerung_1784-page-region/data/mets.xml -I OCR-D-IMG -O BIN -P model default -P operation_level region ocrd-sbb-binarize -m test/assets/kant_aufklaerung_1784/data/mets.xml -I OCR-D-IMG -O BIN -P model default
ocrd-sbb-binarize -m test/assets/kant_aufklaerung_1784/data/mets.xml -I OCR-D-IMG -O BIN2 -P model default-2021-03-09
ocrd-sbb-binarize -m test/assets/kant_aufklaerung_1784-page-region/data/mets.xml -g phys_0001 -I OCR-D-GT-SEG-REGION -O BIN -P model default -P operation_level region
ocrd-sbb-binarize -m test/assets/kant_aufklaerung_1784-page-region/data/mets.xml -g phys_0001 -I OCR-D-GT-SEG-REGION -O BIN2 -P model default-2021-03-09 -P operation_level region
.PHONY: clean
clean:
-$(RM) -fr test/assets

@ -2,6 +2,10 @@
> Binarization for document images > Binarization for document images
[![pip release](https://img.shields.io/pypi/v/sbb-binarization.svg)](https://pypi.org/project/sbb-binarization/)
[![CircleCI test](https://circleci.com/gh/qurator-spk/sbb_binarization.svg?style=svg)](https://circleci.com/gh/qurator-spk/sbb_binarization)
[![GHAction test](https://github.com/qurator-spk/sbb_binarization/actions/workflows/test.yml/badge.svg)](https://github.com/qurator-spk/sbb_binarization/actions/workflows/test.yml)
## Examples ## Examples
<img src="https://user-images.githubusercontent.com/952378/63592437-e433e400-c5b1-11e9-9c2d-889c6e93d748.jpg" width="180"><img src="https://user-images.githubusercontent.com/952378/63592435-e433e400-c5b1-11e9-88e4-3e441b61fa67.jpg" width="180"><img src="https://user-images.githubusercontent.com/952378/63592440-e4cc7a80-c5b1-11e9-8964-2cd1b22c87be.jpg" width="220"><img src="https://user-images.githubusercontent.com/952378/63592438-e4cc7a80-c5b1-11e9-86dc-a9e9f8555422.jpg" width="220"> <img src="https://user-images.githubusercontent.com/952378/63592437-e433e400-c5b1-11e9-9c2d-889c6e93d748.jpg" width="180"><img src="https://user-images.githubusercontent.com/952378/63592435-e433e400-c5b1-11e9-88e4-3e441b61fa67.jpg" width="180"><img src="https://user-images.githubusercontent.com/952378/63592440-e4cc7a80-c5b1-11e9-8964-2cd1b22c87be.jpg" width="220"><img src="https://user-images.githubusercontent.com/952378/63592438-e4cc7a80-c5b1-11e9-86dc-a9e9f8555422.jpg" width="220">
@ -18,13 +22,19 @@ Clone the repository, enter it and run
### Models ### Models
Pre-trained models in `HDF5` format can be downloaded from here: Pre-trained models in HDF5 format can be downloaded from here:
https://qurator-data.de/sbb_binarization/ https://qurator-data.de/sbb_binarization/
We also provide a Tensorflow `saved_model` via Huggingface: We also provide models in Tensorflow SavedModel format via Huggingface and Github release assets:
https://huggingface.co/SBB/sbb_binarization https://huggingface.co/SBB/sbb_binarization
https://github.com/qurator-spk/sbb_binarization/releases
With [OCR-D](https://ocr-d.de/), you can use the [Resource Manager](Tensorflow SavedModel) to deploy models, e.g.
ocrd resmgr download ocrd-sbb-binarize "*"
## Usage ## Usage
@ -39,11 +49,22 @@ Images containing a lot of border noise (black pixels) should be cropped beforeh
### Example ### Example
```sh
sbb_binarize -m /path/to/model/ myimage.tif myimage-bin.tif sbb_binarize -m /path/to/model/ myimage.tif myimage-bin.tif
```
To use the [OCR-D](https://ocr-d.de/) interface:
```sh To use the [OCR-D](https://ocr-d.de/en/spec/cli) interface:
ocrd-sbb-binarize --overwrite -I INPUT_FILE_GRP -O OCR-D-IMG-BIN -P model "/var/lib/sbb_binarization"
``` ocrd-sbb-binarize -I INPUT_FILE_GRP -O OCR-D-IMG-BIN -P model default
## Testing
For simple smoke tests, the following will
- download models
- download test data
- run the OCR-D wrapper (on page and region level):
make model
make test

@ -6,7 +6,7 @@ from click import command, option, argument, version_option, types
from .sbb_binarize import SbbBinarizer from .sbb_binarize import SbbBinarizer
@command() @command()
@version_option() @version_option(package_name="sbb-binarization")
@option('--model-dir', '-m', type=types.Path(exists=True, file_okay=False), required=True, help='directory containing models for prediction') @option('--model-dir', '-m', type=types.Path(exists=True, file_okay=False), required=True, help='directory containing models for prediction')
@argument('input_image') @argument('input_image')
@argument('output_image') @argument('output_image')

@ -26,20 +26,20 @@
}, },
"resources": [ "resources": [
{ {
"url": "https://github.com/apacha/sbb_binarization/releases/download/pre-trained-models/model_2020_01_16.zip", "url": "https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2020_01_16.zip",
"name": "default", "name": "default",
"type": "archive", "type": "archive",
"path_in_archive": "model_2020_01_16", "path_in_archive": "saved_model_2020_01_16",
"size": 562917559, "size": 563147331,
"description": "default models provided by github.com/qurator-spk" "description": "default models provided by github.com/qurator-spk (SavedModel format)"
}, },
{ {
"url": "https://github.com/apacha/sbb_binarization/releases/download/pre-trained-models/model_2021_03_09.zip", "url": "https://github.com/qurator-spk/sbb_binarization/releases/download/v0.0.11/saved_model_2021_03_09.zip",
"name": "default-2021-03-09", "name": "default-2021-03-09",
"type": "archive", "type": "archive",
"path_in_archive": ".", "path_in_archive": ".",
"size": 133693693, "size": 133230419,
"description": "updated default models provided by github.com/qurator-spk" "description": "updated default models provided by github.com/qurator-spk (SavedModel format)"
} }
] ]
} }

@ -124,7 +124,7 @@ class SbbBinarizeProcessor(Processor):
LOG.warning("Page '%s' contains no text/table regions", page_id) LOG.warning("Page '%s' contains no text/table regions", page_id)
for region in regions: for region in regions:
region_image, region_xywh = self.workspace.image_from_segment(region, page_image, page_xywh, feature_filter='binarized') region_image, region_xywh = self.workspace.image_from_segment(region, page_image, page_xywh, feature_filter='binarized')
region_image_bin = cv2pil(binarizer.run(image=pil2cv(region_image))) region_image_bin = cv2pil(self.binarizer.run(image=pil2cv(region_image)))
region_image_bin_path = self.workspace.save_image_file( region_image_bin_path = self.workspace.save_image_file(
region_image_bin, region_image_bin,
"%s_%s.IMG-BIN" % (file_id, region.id), "%s_%s.IMG-BIN" % (file_id, region.id),
@ -139,7 +139,7 @@ class SbbBinarizeProcessor(Processor):
LOG.warning("Page '%s' contains no text lines", page_id) LOG.warning("Page '%s' contains no text lines", page_id)
for region_id, line in region_line_tuples: for region_id, line in region_line_tuples:
line_image, line_xywh = self.workspace.image_from_segment(line, page_image, page_xywh, feature_filter='binarized') line_image, line_xywh = self.workspace.image_from_segment(line, page_image, page_xywh, feature_filter='binarized')
line_image_bin = cv2pil(binarizer.run(image=pil2cv(line_image))) line_image_bin = cv2pil(self.binarizer.run(image=pil2cv(line_image)))
line_image_bin_path = self.workspace.save_image_file( line_image_bin_path = self.workspace.save_image_file(
line_image_bin, line_image_bin,
"%s_%s_%s.IMG-BIN" % (file_id, region_id, line.id), "%s_%s_%s.IMG-BIN" % (file_id, region_id, line.id),

@ -255,9 +255,11 @@ class SbbBinarizer:
img_fin = (res[:, :] == 0) * 255 img_fin = (res[:, :] == 0) * 255
img_last = img_last + img_fin img_last = img_last + img_fin
kernel = np.ones((5, 5), np.uint8)
img_last[:, :][img_last[:, :] > 0] = 255 img_last[:, :][img_last[:, :] > 0] = 255
img_last = (img_last[:, :] == 0) * 255 img_last = (img_last[:, :] == 0) * 255
if save: if save:
cv2.imwrite(save, img_last) cv2.imwrite(save, img_last.astype(np.uint8), [
cv2.IMWRITE_PNG_BILEVEL, 1,
cv2.IMWRITE_JPEG_QUALITY, 100
])
return img_last return img_last

@ -17,7 +17,7 @@ setup(
author='Vahid Rezanezhad', author='Vahid Rezanezhad',
url='https://github.com/qurator-spk/sbb_binarization', url='https://github.com/qurator-spk/sbb_binarization',
license='Apache License 2.0', license='Apache License 2.0',
packages=find_packages(exclude=('tests', 'docs')), packages=find_packages(exclude=('test', 'repo')),
include_package_data=True, include_package_data=True,
package_data={'': ['*.json', '*.yml', '*.yaml']}, package_data={'': ['*.json', '*.yml', '*.yaml']},
install_requires=install_requires, install_requires=install_requires,

Loading…
Cancel
Save