From 131e862762f5a43451ac7e6272a780f79e12d21b Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 30 Jul 2020 18:02:56 +0200 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=92=A9=20Travis:=20Skip=20validation?= =?UTF-8?q?=20until=20https://github.com/OCR-D/ocrd=5Folena/issues/60=20is?= =?UTF-8?q?=20fixed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 97e00f2..bb23a6d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,8 @@ jobs: - unzip actevedef_718448162.first-page.zip - cd actevedef_718448162.first-page - - $TRAVIS_BUILD_DIR/run + - $TRAVIS_BUILD_DIR/run --skip-validation + # XXX Enable validation again when ocrd_olena is fixed https://github.com/OCR-D/ocrd_olena/issues/60 - curl --upload-file OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml https://transfer.sh/my_ocrd_workflow_$RANDOM.xml 1>&2 - grep -q 'auswa.rtige Rechtsgela.hrte ha.tten mu...en bef.aget werden' OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml From 0f8b2d82d577c45c6413b536408de1a0aedc724a Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 30 Jul 2020 19:12:08 +0200 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=A7=B9=20Travis:=20Comment=20out=20tr?= =?UTF-8?q?ansfer.sh=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index bb23a6d..af9a8d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ jobs: - $TRAVIS_BUILD_DIR/run --skip-validation # XXX Enable validation again when ocrd_olena is fixed https://github.com/OCR-D/ocrd_olena/issues/60 - - curl --upload-file OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml https://transfer.sh/my_ocrd_workflow_$RANDOM.xml 1>&2 + #- curl --upload-file OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml https://transfer.sh/my_ocrd_workflow_$RANDOM.xml 1>&2 - grep -q 'auswa.rtige Rechtsgela.hrte ha.tten mu...en bef.aget werden' OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml - grep -q 'auswa.rtige Rechtsgela.hrte ha.tten mu...en bef.aget werden' OCR-D-OCR-TESS/OCR-D-OCR-TESS_00000024.xml From 3af3c6dd00b7d6e42bf44002f370c61de2db88d0 Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 30 Jul 2020 19:23:45 +0200 Subject: [PATCH 3/7] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20Update=20qurator=5Fdat?= =?UTF-8?q?a=5Flib.sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qurator_data_lib.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qurator_data_lib.sh b/qurator_data_lib.sh index d657667..c79124a 100644 --- a/qurator_data_lib.sh +++ b/qurator_data_lib.sh @@ -23,8 +23,8 @@ check_data_subdir() { if ! [ -e $DATA_SUBDIR/.git/annex ]; then echo "$DATA_SUBDIR/ is not a git annex repository"; result=1 fi - if ! (cd $DATA_SUBDIR && git annex version | grep -q 'local repository version: 7'); then - echo "$DATA_SUBDIR/ is not a git annex repository version 7"; result=1 + if ! (cd $DATA_SUBDIR && git annex version | egrep -q 'local repository version: (7|8)'); then + echo "$DATA_SUBDIR/ is not a git annex repository version 7 or 8"; result=1 fi if ! (cd $DATA_SUBDIR && git remote | grep -q '^nfs$'); then echo "$DATA_SUBDIR/ has no git remote 'nfs'"; result=1 @@ -82,7 +82,7 @@ suggest_commands() { echo echo "git submodule update --init" echo "(cd $DATA_SUBDIR && git annex init --version=7)" - echo "(cd $DATA_SUBDIR && git remote add nfs /<... path to ...>/GitNX-Repository/qurator/qurator-data)" + echo "(cd $DATA_SUBDIR && git remote add nfs annex@b-lx0053.sbb.spk-berlin.de:/var/lib/annex/qurator-data.git)" } handle_data() { From efd955c04f4e1e78103764e3fbae818f9fdd2271 Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 30 Jul 2020 20:20:52 +0200 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=A7=B9=20Modernize=20my=5Focrd=5Fwork?= =?UTF-8?q?flow=20and=20use=20OCR-D's=20new=20--overwrite?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my_ocrd_workflow | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/my_ocrd_workflow b/my_ocrd_workflow index 86b59a1..18c096d 100755 --- a/my_ocrd_workflow +++ b/my_ocrd_workflow @@ -59,18 +59,14 @@ do_binarization() { ocrd_olena_binarize_parameters='{ "impl": "sauvola-ms-split" }' - ocrd workspace remove-group -rf OCR-D-IMG-BINPAGE - ocrd workspace remove-group -rf OCR-D-IMG-BIN - ocrd-olena-binarize -I $INPUT_FILE_GRP -O OCR-D-IMG-BINPAGE,OCR-D-IMG-BIN -p "$ocrd_olena_binarize_parameters" + ocrd-olena-binarize --overwrite -I $INPUT_FILE_GRP -O OCR-D-IMG-BINPAGE,OCR-D-IMG-BIN -p "$ocrd_olena_binarize_parameters" } do_linesegmentation_tesserocr() { # Segment the lines in the binarized images - ocrd workspace remove-group -rf OCR-D-SEG-REGION - ocrd workspace remove-group -rf OCR-D-SEG-LINE - ocrd-tesserocr-segment-region -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-REGION - ocrd-tesserocr-segment-line -I OCR-D-SEG-REGION -O OCR-D-SEG-LINE + ocrd-tesserocr-segment-region --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-REGION + ocrd-tesserocr-segment-line --overwrite -I OCR-D-SEG-REGION -O OCR-D-SEG-LINE } do_linesegmentation_sbb() { @@ -79,9 +75,7 @@ do_linesegmentation_sbb() { ocrd_sbb_textline_detector_parameters='{ "model": "/var/lib/textline_detection" }' - ocrd workspace remove-group -rf OCR-D-SEG-REGION - ocrd workspace remove-group -rf OCR-D-SEG-LINE - ocrd-sbb-textline-detector -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-LINE -p "$ocrd_sbb_textline_detector_parameters" + ocrd-sbb-textline-detector --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-LINE -p "$ocrd_sbb_textline_detector_parameters" } do_ocr() { @@ -91,8 +85,7 @@ do_ocr() { "model": "GT4HistOCR_2000000", "textequiv_level": "'$TEXTEQUIV_LEVEL'" }' - ocrd workspace remove-group -rf OCR-D-OCR-TESS - ocrd-tesserocr-recognize -I OCR-D-SEG-LINE -O OCR-D-OCR-TESS -p "$ocrd_tesserocr_recognize_parameters" + ocrd-tesserocr-recognize --overwrite -I OCR-D-SEG-LINE -O OCR-D-OCR-TESS -p "$ocrd_tesserocr_recognize_parameters" } do_ocr_calamari() { @@ -102,8 +95,7 @@ do_ocr_calamari() { "checkpoint": "/var/lib/calamari-models/GT4HistOCR/2019-07-22T15:49+0200/*.ckpt.json", "textequiv_level": "'$TEXTEQUIV_LEVEL'" }' - ocrd workspace remove-group -rf OCR-D-OCR-CALAMARI - ocrd-calamari-recognize -I OCR-D-SEG-LINE -O OCR-D-OCR-CALAMARI -p "$ocrd_calamari_recognize_parameters" + ocrd-calamari-recognize --overwrite -I OCR-D-SEG-LINE -O OCR-D-OCR-CALAMARI -p "$ocrd_calamari_recognize_parameters" } page_validate_xml() { @@ -159,8 +151,7 @@ main() { do_validate if ocrd workspace list-group | grep -q OCR-D-GT-PAGE; then - ocrd workspace remove-group -rf $ocr_filegrp-EVAL - ocrd-dinglehopper -I OCR-D-GT-PAGE,$ocr_filegrp -O $ocr_filegrp-EVAL + ocrd-dinglehopper --overwrite -I OCR-D-GT-PAGE,$ocr_filegrp -O $ocr_filegrp-EVAL fi done From 1a308a5522195aa2743e8fdadd1826ccd1eb8257 Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 30 Jul 2020 20:55:11 +0200 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=A7=B9=20Use=20OCR-D's=20-P,=20remove?= =?UTF-8?q?=20now=20redundant=20validation=20and=20remove=20now=20unnecess?= =?UTF-8?q?ary=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my_ocrd_workflow | 93 +++++------------------------------------------- 1 file changed, 8 insertions(+), 85 deletions(-) diff --git a/my_ocrd_workflow b/my_ocrd_workflow index 18c096d..833b0cd 100755 --- a/my_ocrd_workflow +++ b/my_ocrd_workflow @@ -53,107 +53,30 @@ do_validate() { fi } -do_binarization() { - # Binarize the images - - ocrd_olena_binarize_parameters='{ - "impl": "sauvola-ms-split" - }' - ocrd-olena-binarize --overwrite -I $INPUT_FILE_GRP -O OCR-D-IMG-BINPAGE,OCR-D-IMG-BIN -p "$ocrd_olena_binarize_parameters" -} - -do_linesegmentation_tesserocr() { - # Segment the lines in the binarized images - - ocrd-tesserocr-segment-region --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-REGION - ocrd-tesserocr-segment-line --overwrite -I OCR-D-SEG-REGION -O OCR-D-SEG-LINE -} - -do_linesegmentation_sbb() { - # Segment the lines in the images - - ocrd_sbb_textline_detector_parameters='{ - "model": "/var/lib/textline_detection" - }' - ocrd-sbb-textline-detector --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-LINE -p "$ocrd_sbb_textline_detector_parameters" -} - -do_ocr() { - # Perform OCR on the segmented lines - - ocrd_tesserocr_recognize_parameters='{ - "model": "GT4HistOCR_2000000", - "textequiv_level": "'$TEXTEQUIV_LEVEL'" - }' - ocrd-tesserocr-recognize --overwrite -I OCR-D-SEG-LINE -O OCR-D-OCR-TESS -p "$ocrd_tesserocr_recognize_parameters" -} - -do_ocr_calamari() { - # Perform OCR on the segmented lines - - ocrd_calamari_recognize_parameters='{ - "checkpoint": "/var/lib/calamari-models/GT4HistOCR/2019-07-22T15:49+0200/*.ckpt.json", - "textequiv_level": "'$TEXTEQUIV_LEVEL'" - }' - ocrd-calamari-recognize --overwrite -I OCR-D-SEG-LINE -O OCR-D-OCR-CALAMARI -p "$ocrd_calamari_recognize_parameters" -} - -page_validate_xml() { - # Validate all PAGE XML against the XML schema - - filegrp=$1 - - local file - for file in `ocrd workspace find -G $filegrp`; do - XSD_DIR=`dirname $0`/xsd - if [ ! -d "$XSD_DIR" ]; then - XSD_DIR=/usr/share/xml - fi - xmllint --noout --schema $XSD_DIR/pagecontent.2019-07-15.xsd $file - done -} - -page_upgrade_to_2019() { - # Upgrade PAGE files to 2019-07-15 - - filegrp=$1 - - local file - for file in `ocrd workspace find -G $filegrp`; do - sed -i 's#pagecontent/[0-9-]*#pagecontent/2019-07-15#g' $file - done -} - main() { do_validate - do_binarization - do_validate - - do_linesegmentation_sbb - page_upgrade_to_2019 OCR-D-SEG-LINE - page_validate_xml OCR-D-SEG-REGION - page_validate_xml OCR-D-SEG-LINE + ocrd-olena-binarize --overwrite -I $INPUT_FILE_GRP -O OCR-D-IMG-BINPAGE,OCR-D-IMG-BIN -P impl "sauvola-ms-split" do_validate - do_ocr_calamari + #ocrd-tesserocr-segment-region --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-REGION + #ocrd-tesserocr-segment-line --overwrite -I OCR-D-SEG-REGION -O OCR-D-SEG-LINE + ocrd-sbb-textline-detector --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-LINE -P model "/var/lib/textline_detection" + do_validate - do_ocr + ocrd-calamari-recognize --overwrite -I OCR-D-SEG-LINE -O OCR-D-OCR-CALAMARI -P checkpoint "/var/lib/calamari-models/GT4HistOCR/2019-07-22T15:49+0200/*.ckpt.json" -P textequiv_level "$TEXTEQUIV_LEVEL" + ocrd-tesserocr-recognize --overwrite -I OCR-D-SEG-LINE -O OCR-D-OCR-TESS -P model "GT4HistOCR_2000000" -P textequiv_level "$TEXTEQUIV_LEVEL" + do_validate for ocr_filegrp in OCR-D-OCR-CALAMARI OCR-D-OCR-TESS; do - - page_validate_xml $ocr_filegrp - do_validate - if ocrd workspace list-group | grep -q OCR-D-GT-PAGE; then ocrd-dinglehopper --overwrite -I OCR-D-GT-PAGE,$ocr_filegrp -O $ocr_filegrp-EVAL fi - done } From 7111d28f9b9400c3391862b531d22dfb2e1e827b Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Fri, 31 Jul 2020 13:52:21 +0200 Subject: [PATCH 6/7] =?UTF-8?q?=E2=AC=86=EF=B8=8F=20ocrd=5Folena=20?= =?UTF-8?q?=E2=86=92=201.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 3 +-- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index af9a8d9..582aadc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,8 +39,7 @@ jobs: - unzip actevedef_718448162.first-page.zip - cd actevedef_718448162.first-page - - $TRAVIS_BUILD_DIR/run --skip-validation - # XXX Enable validation again when ocrd_olena is fixed https://github.com/OCR-D/ocrd_olena/issues/60 + - $TRAVIS_BUILD_DIR/run #- curl --upload-file OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml https://transfer.sh/my_ocrd_workflow_$RANDOM.xml 1>&2 - grep -q 'auswa.rtige Rechtsgela.hrte ha.tten mu...en bef.aget werden' OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_00000024.xml diff --git a/Dockerfile b/Dockerfile index 84de53b..3003cef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:18.04 ENV LC_ALL=C.UTF-8 LANG=C.UTF-8 -ENV OCRD_OLENA_VERSION 1.1.10 +ENV OCRD_OLENA_VERSION 1.2.0 ENV TESSDATA_BEST_VERSION 4.0.0 ENV TESSDATA_PREFIX /usr/local/share/tessdata From d1a2bfe669aa612a9fc82b72c63bfb1b1220e536 Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Fri, 31 Jul 2020 14:25:35 +0200 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=90=9B=20Deal=20with=20ocrd=5Folena?= =?UTF-8?q?=20>=3D=201.2.0=20using=20one=20output=20file=20group=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my_ocrd_workflow | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/my_ocrd_workflow b/my_ocrd_workflow index 833b0cd..cabc80d 100755 --- a/my_ocrd_workflow +++ b/my_ocrd_workflow @@ -58,13 +58,13 @@ main() { do_validate - ocrd-olena-binarize --overwrite -I $INPUT_FILE_GRP -O OCR-D-IMG-BINPAGE,OCR-D-IMG-BIN -P impl "sauvola-ms-split" + ocrd-olena-binarize --overwrite -I $INPUT_FILE_GRP -O OCR-D-IMG-BIN -P impl "sauvola-ms-split" do_validate - #ocrd-tesserocr-segment-region --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-REGION + #ocrd-tesserocr-segment-region --overwrite -I OCR-D-IMG-BIN -O OCR-D-SEG-REGION #ocrd-tesserocr-segment-line --overwrite -I OCR-D-SEG-REGION -O OCR-D-SEG-LINE - ocrd-sbb-textline-detector --overwrite -I OCR-D-IMG-BINPAGE -O OCR-D-SEG-LINE -P model "/var/lib/textline_detection" + ocrd-sbb-textline-detector --overwrite -I OCR-D-IMG-BIN -O OCR-D-SEG-LINE -P model "/var/lib/textline_detection" do_validate