|
|
|
@ -14,8 +14,7 @@ from .base import assets
|
|
|
|
|
|
|
|
|
|
|
|
METS_KANT = assets.url_of('kant_aufklaerung_1784-page-region-line-word_glyph/data/mets.xml')
|
|
|
|
METS_KANT = assets.url_of('kant_aufklaerung_1784-page-region-line-word_glyph/data/mets.xml')
|
|
|
|
WORKSPACE_DIR = '/tmp/test-ocrd-calamari'
|
|
|
|
WORKSPACE_DIR = '/tmp/test-ocrd-calamari'
|
|
|
|
CHECKPOINT_DIR = os.path.join(os.getcwd(), 'gt4histocr-calamari1')
|
|
|
|
CHECKPOINT_DIR = os.getenv('MODEL')
|
|
|
|
CHECKPOINT = os.path.join(CHECKPOINT_DIR, '*.ckpt.json')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Because XML namespace versions are so much fun, we not only use one, we use TWO!
|
|
|
|
# Because XML namespace versions are so much fun, we not only use one, we use TWO!
|
|
|
|
NSMAP = { "pc": "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15" }
|
|
|
|
NSMAP = { "pc": "http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15" }
|
|
|
|
@ -31,14 +30,6 @@ def workspace():
|
|
|
|
resolver = Resolver()
|
|
|
|
resolver = Resolver()
|
|
|
|
workspace = resolver.workspace_from_url(METS_KANT, dst_dir=WORKSPACE_DIR)
|
|
|
|
workspace = resolver.workspace_from_url(METS_KANT, dst_dir=WORKSPACE_DIR)
|
|
|
|
|
|
|
|
|
|
|
|
# XXX Work around data bug(?):
|
|
|
|
|
|
|
|
# PAGE-XML links to OCR-D-IMG/INPUT_0017.tif, but this is nothing core can download
|
|
|
|
|
|
|
|
os.makedirs(os.path.join(WORKSPACE_DIR, 'OCR-D-IMG'))
|
|
|
|
|
|
|
|
for f in ['INPUT_0017.tif', 'INPUT_0020.tif']:
|
|
|
|
|
|
|
|
urllib.request.urlretrieve(
|
|
|
|
|
|
|
|
"https://github.com/OCR-D/assets/raw/master/data/kant_aufklaerung_1784/data/OCR-D-IMG/" + f,
|
|
|
|
|
|
|
|
os.path.join(WORKSPACE_DIR, 'OCR-D-IMG', f))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# The binarization options I have are:
|
|
|
|
# The binarization options I have are:
|
|
|
|
#
|
|
|
|
#
|
|
|
|
# a. ocrd_kraken which tries to install cltsm, whose installation is borken on my machine (protobuf)
|
|
|
|
# a. ocrd_kraken which tries to install cltsm, whose installation is borken on my machine (protobuf)
|
|
|
|
@ -47,21 +38,22 @@ def workspace():
|
|
|
|
# c. just fumble with the original files
|
|
|
|
# c. just fumble with the original files
|
|
|
|
#
|
|
|
|
#
|
|
|
|
# So I'm going for option c.
|
|
|
|
# So I'm going for option c.
|
|
|
|
for f in ['INPUT_0017.tif', 'INPUT_0020.tif']:
|
|
|
|
for imgf in workspace.mets.find_files(fileGrp="OCR-D-IMG"):
|
|
|
|
ff = os.path.join(WORKSPACE_DIR, 'OCR-D-IMG', f)
|
|
|
|
imgf = workspace.download_file(imgf)
|
|
|
|
subprocess.call(['convert', ff, '-threshold', '50%', ff])
|
|
|
|
path = os.path.join(workspace.directory, imgf.local_filename)
|
|
|
|
|
|
|
|
subprocess.call(['mogrify', '-threshold', '50%', path])
|
|
|
|
|
|
|
|
|
|
|
|
# Remove GT Words and TextEquivs, to not accidently check GT text instead of the OCR text
|
|
|
|
# Remove GT Words and TextEquivs, to not accidently check GT text instead of the OCR text
|
|
|
|
# XXX Review data again
|
|
|
|
# XXX Review data again
|
|
|
|
# XXX Make this more robust against namespace version changes
|
|
|
|
# XXX Make this more robust against namespace version changes
|
|
|
|
for of in workspace.mets.find_files(fileGrp="OCR-D-GT-SEG-LINE"):
|
|
|
|
for of in workspace.mets.find_files(fileGrp="OCR-D-GT-SEG-WORD-GLYPH"):
|
|
|
|
workspace.download_file(of)
|
|
|
|
workspace.download_file(of)
|
|
|
|
for to_remove in ["//pc:Word", "//pc:TextEquiv"]:
|
|
|
|
path = os.path.join(workspace.directory, of.local_filename)
|
|
|
|
for ff in glob(os.path.join(WORKSPACE_DIR, "OCR-D-GT-SEG-LINE", "*")):
|
|
|
|
tree = etree.parse(path)
|
|
|
|
tree = etree.parse(ff)
|
|
|
|
for to_remove in ["//pc:Word", "//pc:TextEquiv"]:
|
|
|
|
for e in tree.xpath(to_remove, namespaces=NSMAP_GT):
|
|
|
|
for e in tree.xpath(to_remove, namespaces=NSMAP_GT):
|
|
|
|
e.getparent().remove(e)
|
|
|
|
e.getparent().remove(e)
|
|
|
|
tree.write(ff, xml_declaration=True, encoding="utf-8")
|
|
|
|
tree.write(path, xml_declaration=True, encoding="utf-8")
|
|
|
|
|
|
|
|
|
|
|
|
return workspace
|
|
|
|
return workspace
|
|
|
|
|
|
|
|
|
|
|
|
@ -69,23 +61,7 @@ def workspace():
|
|
|
|
def test_recognize(workspace):
|
|
|
|
def test_recognize(workspace):
|
|
|
|
CalamariRecognize(
|
|
|
|
CalamariRecognize(
|
|
|
|
workspace,
|
|
|
|
workspace,
|
|
|
|
input_file_grp="OCR-D-GT-SEG-LINE",
|
|
|
|
input_file_grp="OCR-D-GT-SEG-WORD-GLYPH",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
|
|
|
|
parameter={
|
|
|
|
|
|
|
|
"checkpoint": CHECKPOINT,
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
).process()
|
|
|
|
|
|
|
|
workspace.save_mets()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
page1 = os.path.join(workspace.directory, "OCR-D-OCR-CALAMARI/OCR-D-OCR-CALAMARI_0001.xml")
|
|
|
|
|
|
|
|
assert os.path.exists(page1)
|
|
|
|
|
|
|
|
with open(page1, "r", encoding="utf-8") as f:
|
|
|
|
|
|
|
|
assert "verſchuldeten" in f.read()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_recognize_with_checkpoint_dir(workspace):
|
|
|
|
|
|
|
|
CalamariRecognize(
|
|
|
|
|
|
|
|
workspace,
|
|
|
|
|
|
|
|
input_file_grp="OCR-D-GT-SEG-LINE",
|
|
|
|
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
parameter={
|
|
|
|
parameter={
|
|
|
|
"checkpoint_dir": CHECKPOINT_DIR,
|
|
|
|
"checkpoint_dir": CHECKPOINT_DIR,
|
|
|
|
@ -103,9 +79,9 @@ def test_recognize_should_warn_if_given_rgb_image_and_single_channel_model(works
|
|
|
|
caplog.set_level(logging.WARNING)
|
|
|
|
caplog.set_level(logging.WARNING)
|
|
|
|
CalamariRecognize(
|
|
|
|
CalamariRecognize(
|
|
|
|
workspace,
|
|
|
|
workspace,
|
|
|
|
input_file_grp="OCR-D-GT-SEG-LINE",
|
|
|
|
input_file_grp="OCR-D-GT-SEG-WORD-GLYPH",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI-BROKEN",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI-BROKEN",
|
|
|
|
parameter={'checkpoint': CHECKPOINT}
|
|
|
|
parameter={'checkpoint_dir': CHECKPOINT_DIR}
|
|
|
|
).process()
|
|
|
|
).process()
|
|
|
|
|
|
|
|
|
|
|
|
interesting_log_messages = [t[2] for t in caplog.record_tuples if "Using raw image" in t[2]]
|
|
|
|
interesting_log_messages = [t[2] for t in caplog.record_tuples if "Using raw image" in t[2]]
|
|
|
|
@ -115,10 +91,10 @@ def test_recognize_should_warn_if_given_rgb_image_and_single_channel_model(works
|
|
|
|
def test_word_segmentation(workspace):
|
|
|
|
def test_word_segmentation(workspace):
|
|
|
|
CalamariRecognize(
|
|
|
|
CalamariRecognize(
|
|
|
|
workspace,
|
|
|
|
workspace,
|
|
|
|
input_file_grp="OCR-D-GT-SEG-LINE",
|
|
|
|
input_file_grp="OCR-D-GT-SEG-WORD-GLYPH",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
parameter={
|
|
|
|
parameter={
|
|
|
|
"checkpoint": CHECKPOINT,
|
|
|
|
"checkpoint_dir": CHECKPOINT_DIR,
|
|
|
|
"textequiv_level": "word", # Note that we're going down to word level here
|
|
|
|
"textequiv_level": "word", # Note that we're going down to word level here
|
|
|
|
}
|
|
|
|
}
|
|
|
|
).process()
|
|
|
|
).process()
|
|
|
|
@ -147,10 +123,10 @@ def test_word_segmentation(workspace):
|
|
|
|
def test_glyphs(workspace):
|
|
|
|
def test_glyphs(workspace):
|
|
|
|
CalamariRecognize(
|
|
|
|
CalamariRecognize(
|
|
|
|
workspace,
|
|
|
|
workspace,
|
|
|
|
input_file_grp="OCR-D-GT-SEG-LINE",
|
|
|
|
input_file_grp="OCR-D-GT-SEG-WORD-GLYPH",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
output_file_grp="OCR-D-OCR-CALAMARI",
|
|
|
|
parameter={
|
|
|
|
parameter={
|
|
|
|
"checkpoint": CHECKPOINT,
|
|
|
|
"checkpoint_dir": CHECKPOINT_DIR,
|
|
|
|
"textequiv_level": "glyph", # Note that we're going down to glyph level here
|
|
|
|
"textequiv_level": "glyph", # Note that we're going down to glyph level here
|
|
|
|
}
|
|
|
|
}
|
|
|
|
).process()
|
|
|
|
).process()
|
|
|
|
|
GitHub