mirror of
https://github.com/qurator-spk/page2tsv.git
synced 2025-06-09 19:39:54 +02:00
add alto2tsv
This commit is contained in:
Kai Labusch
parent
0f64f07635
commit
0ec6f83c4c
2 changed files with 185 additions and 0 deletions
|
|
@ -10,6 +10,9 @@ import pandas as pd
|
|||
import requests
|
||||
from lxml import etree as ET
|
||||
|
||||
import xml.etree.ElementTree as ElementTree
|
||||
import unicodedata
|
||||
|
||||
from ocrd_models.ocrd_page import parse
|
||||
from ocrd_utils import bbox_from_points
|
||||
|
||||
|
|
@ -58,6 +61,159 @@ def annotate_tsv(tsv_file, annotated_tsv_file):
|
|||
df.to_csv(annotated_tsv_file, sep="\t", quoting=3, index=False)
|
||||
|
||||
|
||||
def alto_iterate_textblocks(xml_file=None, root=None):
|
||||
|
||||
if root is None:
|
||||
tree = ElementTree.parse(xml_file)
|
||||
root = tree.getroot()
|
||||
|
||||
for idx, block_elem in enumerate(root.iter('{http://www.loc.gov/standards/alto/ns-v2#}TextBlock')):
|
||||
|
||||
id = str(idx)
|
||||
if 'ID' in block_elem.attrib:
|
||||
id = block_elem.attrib['ID']
|
||||
|
||||
yield id, block_elem
|
||||
|
||||
|
||||
def alto_iterate_lines(root):
|
||||
|
||||
for idx, line_elem in enumerate(root.iter('{http://www.loc.gov/standards/alto/ns-v2#}TextLine')):
|
||||
|
||||
left, top, right, bottom = -1, -1, -1, -1
|
||||
|
||||
if 'HPOS' in line_elem.attrib:
|
||||
left = int(line_elem.attrib['HPOS'])
|
||||
|
||||
if 'VPOS' in line_elem.attrib:
|
||||
top = int(line_elem.attrib['VPOS'])
|
||||
|
||||
if 'HPOS' in line_elem.attrib and 'WIDTH' in line_elem.attrib:
|
||||
right = int(line_elem.attrib['HPOS']) + int(line_elem.attrib['WIDTH'])
|
||||
|
||||
if 'VPOS' in line_elem.attrib and 'HEIGHT' in line_elem.attrib:
|
||||
bottom = int(line_elem.attrib['VPOS']) + int(line_elem.attrib['HEIGHT'])
|
||||
|
||||
yield line_elem, str(idx), left, right, top, bottom
|
||||
|
||||
|
||||
def alto_iterate_string_elements(root):
|
||||
|
||||
for string_elem in root.iter('{http://www.loc.gov/standards/alto/ns-v2#}String'):
|
||||
|
||||
if 'CONTENT' in string_elem.attrib:
|
||||
content = string_elem.attrib['CONTENT']
|
||||
else:
|
||||
content = str(np.NAN)
|
||||
|
||||
left, top, right, bottom = -1, -1, -1, -1
|
||||
|
||||
if 'HPOS' in string_elem.attrib:
|
||||
left = int(string_elem.attrib['HPOS'])
|
||||
|
||||
if 'VPOS' in string_elem.attrib:
|
||||
top = int(string_elem.attrib['VPOS'])
|
||||
|
||||
if 'HPOS' in string_elem.attrib and 'WIDTH' in string_elem.attrib:
|
||||
right = int(string_elem.attrib['HPOS']) + int(string_elem.attrib['WIDTH'])
|
||||
|
||||
if 'VPOS' in string_elem.attrib and 'HEIGHT' in string_elem.attrib:
|
||||
bottom = int(string_elem.attrib['VPOS']) + int(string_elem.attrib['HEIGHT'])
|
||||
|
||||
yield unicodedata.normalize('NFC', content), left, top, right, bottom
|
||||
|
||||
|
||||
def alto2tsv(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
|
||||
noproxy, scale_factor, ned_threshold, ned_priority):
|
||||
if purpose == "NERD":
|
||||
out_columns = ['No.', 'TOKEN', 'NE-TAG', 'NE-EMB', 'ID', 'url_id', 'left', 'right', 'top', 'bottom', 'conf']
|
||||
elif purpose == "OCR":
|
||||
out_columns = ['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'conf', 'line_id']
|
||||
|
||||
else:
|
||||
raise RuntimeError("Unknown purpose.")
|
||||
|
||||
if noproxy:
|
||||
os.environ['no_proxy'] = '*'
|
||||
|
||||
urls = []
|
||||
if os.path.exists(tsv_out_file):
|
||||
parts = extract_doc_links(tsv_out_file)
|
||||
urls = [part['url'] for part in parts]
|
||||
else:
|
||||
pd.DataFrame([], columns=out_columns).to_csv(tsv_out_file, sep="\t", quoting=3, index=False)
|
||||
|
||||
tsv = []
|
||||
line_info = []
|
||||
|
||||
for region_idx, region in alto_iterate_textblocks(alto_xml_file):
|
||||
|
||||
for line, line_id, l_left, l_right, l_top, l_bottom in alto_iterate_lines(region):
|
||||
|
||||
line_info.append((len(urls), l_left, l_right, l_top, l_bottom, line_id))
|
||||
|
||||
for word_num, (word, left, top, right, bottom) in enumerate(alto_iterate_string_elements(line)):
|
||||
|
||||
word = word.strip()
|
||||
|
||||
if len(word) == 0:
|
||||
continue
|
||||
|
||||
if len(word.split()) > 1:
|
||||
print(word)
|
||||
continue
|
||||
|
||||
left, top, right, bottom = [int(scale_factor * x) for x in [left, top, right, bottom]]
|
||||
|
||||
tsv.append((region_idx, left + (right - left) / 2.0,
|
||||
word, len(urls), left, right, top, bottom, line_id))
|
||||
|
||||
line_info = pd.DataFrame(line_info, columns=['url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
|
||||
|
||||
tsv = pd.DataFrame(tsv, columns=['rid', 'hcenter'] +
|
||||
['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
|
||||
|
||||
with open(tsv_out_file, 'a') as f:
|
||||
f.write('# ' + image_url + '\n')
|
||||
|
||||
if len(tsv) == 0:
|
||||
return
|
||||
|
||||
vlinecenter = pd.DataFrame(tsv[['line_id', 'top']].groupby('line_id', sort=False).mean().top +
|
||||
(tsv[['line_id', 'bottom']].groupby('line_id', sort=False).mean().bottom -
|
||||
tsv[['line_id', 'top']].groupby('line_id', sort=False).mean().top) / 2,
|
||||
columns=['vlinecenter'])
|
||||
|
||||
tsv = tsv.merge(vlinecenter, left_on='line_id', right_index=True)
|
||||
|
||||
regions = [region.sort_values(['vlinecenter', 'hcenter']) for rid, region in tsv.groupby('rid', sort=False)]
|
||||
tsv = pd.concat(regions)
|
||||
|
||||
if purpose == 'NERD':
|
||||
tsv['No.'] = 0
|
||||
tsv['NE-TAG'] = 'O'
|
||||
tsv['NE-EMB'] = 'O'
|
||||
tsv['ID'] = '-'
|
||||
tsv['conf'] = '-'
|
||||
tsv = tsv.rename(columns={'TEXT': 'TOKEN'})
|
||||
|
||||
elif purpose == 'OCR':
|
||||
tsv = pd.DataFrame([(line_id, " ".join(part.TEXT.to_list())) for line_id, part in tsv.groupby('line_id')],
|
||||
columns=['line_id', 'TEXT'])
|
||||
tsv = tsv.merge(line_info, left_on='line_id', right_index=True)
|
||||
tsv = tsv[out_columns].reset_index(drop=True)
|
||||
|
||||
try:
|
||||
if purpose == 'NERD' and ner_rest_endpoint is not None:
|
||||
tsv, ner_result = ner(tsv, ner_rest_endpoint)
|
||||
if ned_rest_endpoint is not None:
|
||||
tsv, _ = ned(tsv, ner_result, ned_rest_endpoint, threshold=ned_threshold, priority=ned_priority)
|
||||
|
||||
tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
|
||||
except requests.HTTPError as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
|
||||
noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority):
|
||||
if purpose == "NERD":
|
||||
|
|
@ -163,6 +319,7 @@ def page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint,
|
|||
except requests.HTTPError as e:
|
||||
print(e)
|
||||
|
||||
|
||||
def tsv2page(output_filename, keep_words, page_file, tsv_file):
|
||||
if not output_filename:
|
||||
output_filename = Path(page_file).stem + '.corrected.xml'
|
||||
|
|
@ -178,6 +335,7 @@ def tsv2page(output_filename, keep_words, page_file, tsv_file):
|
|||
with open(output_filename, 'w', encoding='utf-8') as f:
|
||||
f.write(ET.tostring(tree, pretty_print=True).decode('utf-8'))
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--output-filename', '-o', help="Output filename. "
|
||||
"If omitted, PAGE-XML filename with .corrected.xml extension")
|
||||
|
|
@ -187,6 +345,7 @@ def tsv2page(output_filename, keep_words, page_file, tsv_file):
|
|||
def tsv2page_cli(output_filename, keep_words, page_file, tsv_file):
|
||||
return tsv2page_cli(output_filename, keep_words, page_file, tsv_file)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--xls-file', type=click.Path(exists=True), default=None,
|
||||
help="Read parameters from xls-file. Expected columns: Filename, iiif_url, scale_factor.")
|
||||
|
|
@ -249,3 +408,28 @@ def page2tsv_cli(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpo
|
|||
noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority):
|
||||
return page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
|
||||
noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument('alto-xml-file', type=click.Path(exists=True), required=True, nargs=1)
|
||||
@click.argument('tsv-out-file', type=click.Path(), required=True, nargs=1)
|
||||
@click.option('--purpose', type=click.Choice(['NERD', 'OCR'], case_sensitive=False), default="NERD",
|
||||
help="Purpose of output tsv file. "
|
||||
"\n\nNERD: NER/NED application/ground-truth creation. "
|
||||
"\n\nOCR: OCR application/ground-truth creation. "
|
||||
"\n\ndefault: NERD.")
|
||||
@click.option('--image-url', type=str, default='http://empty')
|
||||
@click.option('--ner-rest-endpoint', type=str, default=None,
|
||||
help="REST endpoint of sbb_ner service. See https://github.com/qurator-spk/sbb_ner for details. "
|
||||
"Only applicable in case of NERD.")
|
||||
@click.option('--ned-rest-endpoint', type=str, default=None,
|
||||
help="REST endpoint of sbb_ned service. See https://github.com/qurator-spk/sbb_ned for details. "
|
||||
"Only applicable in case of NERD.")
|
||||
@click.option('--noproxy', type=bool, is_flag=True, help='disable proxy. default: enabled.')
|
||||
@click.option('--scale-factor', type=float, default=1.0, help='default: 1.0')
|
||||
@click.option('--ned-threshold', type=float, default=None)
|
||||
@click.option('--ned-priority', type=int, default=1)
|
||||
def alto2tsv_cli(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
|
||||
noproxy, scale_factor, ned_threshold, ned_priority):
|
||||
return alto2tsv(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
|
||||
noproxy, scale_factor, ned_threshold, ned_priority)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue