add alto2tsv

6 months ago · 0ec6f83c4c
parent 0f64f07635
commit 0ec6f83c4c
2 changed files with 185 additions and 0 deletions
--- a/qurator/tsvtools/cli.py
+++ b/qurator/tsvtools/cli.py
@ -10,6 +10,9 @@ import pandas as pd
 import requests
 from lxml import etree as ET

+import xml.etree.ElementTree as ElementTree
+import unicodedata
+
 from ocrd_models.ocrd_page import parse
 from ocrd_utils import bbox_from_points

@ -58,6 +61,159 @@ def annotate_tsv(tsv_file, annotated_tsv_file):
    df.to_csv(annotated_tsv_file, sep="\t", quoting=3, index=False)


+def alto_iterate_textblocks(xml_file=None, root=None):
+
+    if root is None:
+        tree = ElementTree.parse(xml_file)
+        root = tree.getroot()
+
+    for idx, block_elem in enumerate(root.iter('{http://www.loc.gov/standards/alto/ns-v2#}TextBlock')):
+
+        id = str(idx)
+        if 'ID' in block_elem.attrib:
+            id = block_elem.attrib['ID']
+
+        yield id, block_elem
+
+
+def alto_iterate_lines(root):
+
+    for idx, line_elem in enumerate(root.iter('{http://www.loc.gov/standards/alto/ns-v2#}TextLine')):
+
+        left, top, right, bottom = -1, -1, -1, -1
+
+        if 'HPOS' in line_elem.attrib:
+            left = int(line_elem.attrib['HPOS'])
+
+        if 'VPOS' in line_elem.attrib:
+            top = int(line_elem.attrib['VPOS'])
+
+        if 'HPOS' in line_elem.attrib and 'WIDTH' in line_elem.attrib:
+            right = int(line_elem.attrib['HPOS']) + int(line_elem.attrib['WIDTH'])
+
+        if 'VPOS' in line_elem.attrib and 'HEIGHT' in line_elem.attrib:
+            bottom = int(line_elem.attrib['VPOS']) + int(line_elem.attrib['HEIGHT'])
+
+        yield line_elem, str(idx), left, right, top, bottom
+
+
+def alto_iterate_string_elements(root):
+
+    for string_elem in root.iter('{http://www.loc.gov/standards/alto/ns-v2#}String'):
+
+        if 'CONTENT' in string_elem.attrib:
+            content = string_elem.attrib['CONTENT']
+        else:
+            content = str(np.NAN)
+
+        left, top, right, bottom = -1, -1, -1, -1
+
+        if 'HPOS' in string_elem.attrib:
+            left = int(string_elem.attrib['HPOS'])
+
+        if 'VPOS' in string_elem.attrib:
+            top = int(string_elem.attrib['VPOS'])
+
+        if 'HPOS' in string_elem.attrib and 'WIDTH' in string_elem.attrib:
+            right = int(string_elem.attrib['HPOS']) + int(string_elem.attrib['WIDTH'])
+
+        if 'VPOS' in string_elem.attrib and 'HEIGHT' in string_elem.attrib:
+            bottom = int(string_elem.attrib['VPOS']) + int(string_elem.attrib['HEIGHT'])
+
+        yield unicodedata.normalize('NFC', content), left, top, right, bottom
+
+
+def alto2tsv(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
+             noproxy, scale_factor, ned_threshold, ned_priority):
+    if purpose == "NERD":
+        out_columns = ['No.', 'TOKEN', 'NE-TAG', 'NE-EMB', 'ID', 'url_id', 'left', 'right', 'top', 'bottom', 'conf']
+    elif purpose == "OCR":
+        out_columns = ['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'conf', 'line_id']
+
+    else:
+        raise RuntimeError("Unknown purpose.")
+
+    if noproxy:
+        os.environ['no_proxy'] = '*'
+
+    urls = []
+    if os.path.exists(tsv_out_file):
+        parts = extract_doc_links(tsv_out_file)
+        urls = [part['url'] for part in parts]
+    else:
+        pd.DataFrame([], columns=out_columns).to_csv(tsv_out_file, sep="\t", quoting=3, index=False)
+
+    tsv = []
+    line_info = []
+
+    for region_idx, region in alto_iterate_textblocks(alto_xml_file):
+
+        for line, line_id, l_left, l_right, l_top, l_bottom in alto_iterate_lines(region):
+
+            line_info.append((len(urls), l_left, l_right, l_top, l_bottom, line_id))
+
+            for word_num, (word, left, top, right, bottom) in enumerate(alto_iterate_string_elements(line)):
+
+                word = word.strip()
+
+                if len(word) == 0:
+                    continue
+
+                if len(word.split()) > 1:
+                    print(word)
+                    continue
+
+                left, top, right, bottom = [int(scale_factor * x) for x in [left, top, right, bottom]]
+
+                tsv.append((region_idx, left + (right - left) / 2.0,
+                            word, len(urls), left, right, top, bottom, line_id))
+
+    line_info = pd.DataFrame(line_info, columns=['url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
+
+    tsv = pd.DataFrame(tsv, columns=['rid', 'hcenter'] +
+                                    ['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
+
+    with open(tsv_out_file, 'a') as f:
+        f.write('# ' + image_url + '\n')
+
+    if len(tsv) == 0:
+        return
+
+    vlinecenter = pd.DataFrame(tsv[['line_id', 'top']].groupby('line_id', sort=False).mean().top +
+                               (tsv[['line_id', 'bottom']].groupby('line_id', sort=False).mean().bottom -
+                                tsv[['line_id', 'top']].groupby('line_id', sort=False).mean().top) / 2,
+                               columns=['vlinecenter'])
+
+    tsv = tsv.merge(vlinecenter, left_on='line_id', right_index=True)
+
+    regions = [region.sort_values(['vlinecenter', 'hcenter']) for rid, region in tsv.groupby('rid', sort=False)]
+    tsv = pd.concat(regions)
+
+    if purpose == 'NERD':
+        tsv['No.'] = 0
+        tsv['NE-TAG'] = 'O'
+        tsv['NE-EMB'] = 'O'
+        tsv['ID'] = '-'
+        tsv['conf'] = '-'
+        tsv = tsv.rename(columns={'TEXT': 'TOKEN'})
+
+    elif purpose == 'OCR':
+        tsv = pd.DataFrame([(line_id, " ".join(part.TEXT.to_list())) for line_id, part in tsv.groupby('line_id')],
+                           columns=['line_id', 'TEXT'])
+        tsv = tsv.merge(line_info, left_on='line_id', right_index=True)
+    tsv = tsv[out_columns].reset_index(drop=True)
+
+    try:
+        if purpose == 'NERD' and ner_rest_endpoint is not None:
+            tsv, ner_result = ner(tsv, ner_rest_endpoint)
+            if ned_rest_endpoint is not None:
+                tsv, _ = ned(tsv, ner_result, ned_rest_endpoint, threshold=ned_threshold, priority=ned_priority)
+
+        tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
+    except requests.HTTPError as e:
+        print(e)
+
+
 def page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
             noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority):
    if purpose == "NERD":
@ -163,6 +319,7 @@ def page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint,
    except requests.HTTPError as e:
        print(e)

+
 def tsv2page(output_filename, keep_words, page_file, tsv_file):
    if not output_filename:
        output_filename = Path(page_file).stem + '.corrected.xml'
@ -178,6 +335,7 @@ def tsv2page(output_filename, keep_words, page_file, tsv_file):
    with open(output_filename, 'w', encoding='utf-8') as f:
        f.write(ET.tostring(tree, pretty_print=True).decode('utf-8'))

+
@click.command()
@click.option('--output-filename', '-o', help="Output filename. "
                                              "If omitted, PAGE-XML filename with .corrected.xml extension")
@ -187,6 +345,7 @@ def tsv2page(output_filename, keep_words, page_file, tsv_file):
 def tsv2page_cli(output_filename, keep_words, page_file, tsv_file):
    return tsv2page_cli(output_filename, keep_words, page_file, tsv_file)

+
@click.command()
@click.option('--xls-file', type=click.Path(exists=True), default=None,
              help="Read parameters from xls-file. Expected columns:  Filename, iiif_url, scale_factor.")
@ -249,3 +408,28 @@ def page2tsv_cli(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpo
             noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority):
    return page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
             noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority)
+
+
+@click.command()
+@click.argument('alto-xml-file', type=click.Path(exists=True), required=True, nargs=1)
+@click.argument('tsv-out-file', type=click.Path(), required=True, nargs=1)
+@click.option('--purpose', type=click.Choice(['NERD', 'OCR'], case_sensitive=False), default="NERD",
+              help="Purpose of output tsv file. "
+                   "\n\nNERD: NER/NED application/ground-truth creation. "
+                   "\n\nOCR: OCR application/ground-truth creation. "
+                   "\n\ndefault: NERD.")
+@click.option('--image-url', type=str, default='http://empty')
+@click.option('--ner-rest-endpoint', type=str, default=None,
+              help="REST endpoint of sbb_ner service. See https://github.com/qurator-spk/sbb_ner for details. "
+                   "Only applicable in case of NERD.")
+@click.option('--ned-rest-endpoint', type=str, default=None,
+              help="REST endpoint of sbb_ned service. See https://github.com/qurator-spk/sbb_ned for details. "
+                   "Only applicable in case of NERD.")
+@click.option('--noproxy', type=bool, is_flag=True, help='disable proxy. default: enabled.')
+@click.option('--scale-factor', type=float, default=1.0, help='default: 1.0')
+@click.option('--ned-threshold', type=float, default=None)
+@click.option('--ned-priority', type=int, default=1)
+def alto2tsv_cli(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
+             noproxy, scale_factor, ned_threshold, ned_priority):
+    return alto2tsv(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
+             noproxy, scale_factor, ned_threshold, ned_priority)
--- a/setup.py
+++ b/setup.py
@ -33,6 +33,7 @@ setup(
        "ocrd-neat-import=qurator.tsvtools.ocrd_cli:import_cli",
        "page2tsv=qurator.tsvtools.cli:page2tsv_cli",
        "tsv2page=qurator.tsvtools.cli:tsv2page_cli",
+        "alto2tsv=qurator.tsvtools.cli:alto2tsv_cli",
        "make-page2tsv-commands=qurator.tsvtools.cli:make_page2tsv_commands"
      ]
    },