1
0
Fork 0
mirror of https://github.com/qurator-spk/page2tsv.git synced 2025-07-11 02:59:53 +02:00
page2tsv/qurator/tsvtools/cli.py

606 lines
25 KiB
Python
Raw Normal View History

import glob
import re
import os
from io import StringIO
from pathlib import Path
import numpy as np
2021-02-01 16:25:12 +01:00
import click
import pandas as pd
import requests
from lxml import etree as ET
2023-10-20 16:36:43 +02:00
import xml.etree.ElementTree as ElementTree
import unicodedata
from ocrd_models.ocrd_page import parse
from ocrd_utils import bbox_from_points
2021-02-01 16:25:12 +01:00
from qurator.utils.tsv import read_tsv, write_tsv, extract_doc_links
from .ocr import get_conf_color
2021-02-01 16:25:12 +01:00
2022-02-21 14:03:07 +01:00
from qurator.utils.ner import ner
from qurator.utils.ned import ned
2022-06-10 10:00:32 +02:00
2021-02-01 16:25:12 +01:00
@click.command()
@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
@click.argument('url-file', type=click.Path(exists=False), required=True, nargs=1)
def extract_document_links(tsv_file, url_file):
parts = extract_doc_links(tsv_file)
urls = [part['url'] for part in parts]
urls = pd.DataFrame(urls, columns=['url'])
urls.to_csv(url_file, sep="\t", quoting=3, index=False)
@click.command()
@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
@click.argument('annotated-tsv-file', type=click.Path(exists=False), required=True, nargs=1)
def annotate_tsv(tsv_file, annotated_tsv_file):
parts = extract_doc_links(tsv_file)
annotated_parts = []
for part in parts:
part_data = StringIO(part['header'] + part['text'])
df = pd.read_csv(part_data, sep="\t", comment='#', quoting=3)
df['url_id'] = len(annotated_parts)
annotated_parts.append(df)
df = pd.concat(annotated_parts)
df.to_csv(annotated_tsv_file, sep="\t", quoting=3, index=False)
2023-10-20 16:36:43 +02:00
def alto_iterate_textblocks(xml_file=None, root=None):
if root is None:
tree = ElementTree.parse(xml_file)
root = tree.getroot()
for idx, block_elem in enumerate(root.iter('{http://www.loc.gov/standards/alto/ns-v2#}TextBlock')):
id = str(idx)
if 'ID' in block_elem.attrib:
id = block_elem.attrib['ID']
yield id, block_elem
def alto_iterate_lines(root):
for idx, line_elem in enumerate(root.iter('{http://www.loc.gov/standards/alto/ns-v2#}TextLine')):
left, top, right, bottom = -1, -1, -1, -1
if 'HPOS' in line_elem.attrib:
left = int(line_elem.attrib['HPOS'])
if 'VPOS' in line_elem.attrib:
top = int(line_elem.attrib['VPOS'])
if 'HPOS' in line_elem.attrib and 'WIDTH' in line_elem.attrib:
right = int(line_elem.attrib['HPOS']) + int(line_elem.attrib['WIDTH'])
if 'VPOS' in line_elem.attrib and 'HEIGHT' in line_elem.attrib:
bottom = int(line_elem.attrib['VPOS']) + int(line_elem.attrib['HEIGHT'])
yield line_elem, str(idx), left, right, top, bottom
def alto_iterate_string_elements(root):
for string_elem in root.iter('{http://www.loc.gov/standards/alto/ns-v2#}String'):
if 'CONTENT' in string_elem.attrib:
content = string_elem.attrib['CONTENT']
else:
content = str(np.NAN)
left, top, right, bottom = -1, -1, -1, -1
if 'HPOS' in string_elem.attrib:
left = int(string_elem.attrib['HPOS'])
if 'VPOS' in string_elem.attrib:
top = int(string_elem.attrib['VPOS'])
if 'HPOS' in string_elem.attrib and 'WIDTH' in string_elem.attrib:
right = int(string_elem.attrib['HPOS']) + int(string_elem.attrib['WIDTH'])
if 'VPOS' in string_elem.attrib and 'HEIGHT' in string_elem.attrib:
bottom = int(string_elem.attrib['VPOS']) + int(string_elem.attrib['HEIGHT'])
yield unicodedata.normalize('NFC', content), left, top, right, bottom
def alto2tsv(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
noproxy, scale_factor, ned_threshold, ned_priority):
if purpose == "NERD":
out_columns = ['No.', 'TOKEN', 'NE-TAG', 'NE-EMB', 'ID', 'url_id', 'left', 'right', 'top', 'bottom', 'conf']
elif purpose == "OCR":
out_columns = ['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'conf', 'line_id']
else:
raise RuntimeError("Unknown purpose.")
if noproxy:
os.environ['no_proxy'] = '*'
urls = []
if os.path.exists(tsv_out_file):
parts = extract_doc_links(tsv_out_file)
urls = [part['url'] for part in parts]
else:
pd.DataFrame([], columns=out_columns).to_csv(tsv_out_file, sep="\t", quoting=3, index=False)
tsv = []
line_info = []
for region_idx, region in alto_iterate_textblocks(alto_xml_file):
2023-11-10 17:28:30 +01:00
for line, _, l_left, l_right, l_top, l_bottom in alto_iterate_lines(region):
line_id = len(line_info)
2023-10-20 16:36:43 +02:00
line_info.append((len(urls), l_left, l_right, l_top, l_bottom, line_id))
for word_num, (word, left, top, right, bottom) in enumerate(alto_iterate_string_elements(line)):
word = word.strip()
if len(word) == 0:
continue
if len(word.split()) > 1:
print(word)
continue
left, top, right, bottom = [int(scale_factor * x) for x in [left, top, right, bottom]]
tsv.append((region_idx, left + (right - left) / 2.0,
word, len(urls), left, right, top, bottom, line_id))
line_info = pd.DataFrame(line_info, columns=['url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
tsv = pd.DataFrame(tsv, columns=['rid', 'hcenter'] +
['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
with open(tsv_out_file, 'a') as f:
f.write('# ' + image_url + '\n')
if len(tsv) == 0:
return
vlinecenter = pd.DataFrame(tsv[['line_id', 'top']].groupby('line_id', sort=False).mean().top +
(tsv[['line_id', 'bottom']].groupby('line_id', sort=False).mean().bottom -
tsv[['line_id', 'top']].groupby('line_id', sort=False).mean().top) / 2,
columns=['vlinecenter'])
tsv = tsv.merge(vlinecenter, left_on='line_id', right_index=True)
regions = [region.sort_values(['vlinecenter', 'hcenter']) for rid, region in tsv.groupby('rid', sort=False)]
tsv = pd.concat(regions)
if purpose == 'NERD':
tsv['No.'] = 0
tsv['NE-TAG'] = 'O'
tsv['NE-EMB'] = 'O'
tsv['ID'] = '-'
tsv['conf'] = '-'
tsv = tsv.rename(columns={'TEXT': 'TOKEN'})
elif purpose == 'OCR':
tsv = pd.DataFrame([(line_id, " ".join(part.TEXT.to_list())) for line_id, part in tsv.groupby('line_id')],
columns=['line_id', 'TEXT'])
tsv = tsv.merge(line_info, left_on='line_id', right_index=True)
tsv = tsv[out_columns].reset_index(drop=True)
try:
if purpose == 'NERD' and ner_rest_endpoint is not None:
tsv, ner_result = ner(tsv, ner_rest_endpoint)
if ned_rest_endpoint is not None:
tsv, _ = ned(tsv, ner_result, ned_rest_endpoint, threshold=ned_threshold, priority=ned_priority)
tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
except requests.HTTPError as e:
print(e)
def unicode_normalize(text, normalization_map=None, use_combining_characters=True):
if normalization_map is None:
ret = "".join([c if unicodedata.category(c) != "Co" else '' for c in text])
if ret != text:
print("Warning: Due to unicode normalization possible loss of information: "
"{} => {} (normalization file missing?)".format(text, ret))
elif use_combining_characters:
ret = "".join([c if unicodedata.category(c) != "Co" else
"{}{}".format(normalization_map.loc[ord(c)].base,
chr(int(normalization_map.loc[ord(c)].combining_character, base=16))
if normalization_map.loc[ord(c)].combining_character != '' else '')
if ord(c) in normalization_map.index else '' for c in text])
# do it again since the normalization map may again contain unicode private use chars
ret = "".join([c if unicodedata.category(c) != "Co" else '' for c in ret])
if ret != text:
print("Warning: Due to unicode normalization possible loss of information: "
"{} => {}".format(text, ret))
else:
ret = "".join([c if unicodedata.category(c) != "Co" else
normalization_map.loc[ord(c)].base
if ord(c) in normalization_map.index else ''
for c in text])
# do it again since the normalization map may again contain unicode private use chars
ret = "".join([c if unicodedata.category(c) != "Co" else '' for c in ret])
if ret != text:
print("Warning: Due to unicode normalization possible loss of information: "
"{} => {}".format(text, ret))
return unicodedata.normalize('NFC', ret)
2023-10-20 16:36:43 +02:00
@click.command()
@click.argument('tsv-in-file', type=click.Path(exists=True), required=True, nargs=1)
@click.option('--tsv-out-file', type=click.Path(), default=None, help="Write modified TSV to this file.")
@click.option('--ner-rest-endpoint', type=str, default=None, help="REST endpoint of sbb_ner service. "
"See https://github.com/qurator-spk/sbb_ner "
"for details.")
@click.option('--noproxy', type=bool, is_flag=True, help='disable proxy. default: enabled.')
@click.option('--num-tokens', type=bool, is_flag=True, help='Print number of tokens in input/output file.')
@click.option('--sentence-count', type=bool, is_flag=True, help='Print sentence count in input/output file.')
@click.option('--max-sentence-len', type=bool, is_flag=True, help='Print maximum sentence len for input/output file.')
@click.option('--keep-tokenization', type=bool, is_flag=True, help='Keep the word tokenization exactly as it is.')
@click.option('--sentence-split-only', type=bool, is_flag=True, help='Do only sentence splitting.')
@click.option('--show-urls', type=bool, is_flag=True, help='Print contained visualization URLs.')
@click.option('--just-zero', type=bool, is_flag=True, help='Process only files that have max sentence length zero,'
'i.e., that do not have sentence splitting.')
@click.option('--sanitize-sentence-numbers', type=bool, is_flag=True, help='Sanitize sentence numbering.')
def tsv2tsv(tsv_in_file, tsv_out_file, ner_rest_endpoint, noproxy,
num_tokens, sentence_count, max_sentence_len, keep_tokenization, sentence_split_only,
show_urls, just_zero, sanitize_sentence_numbers):
if noproxy:
os.environ['no_proxy'] = '*'
keep_tokenization = keep_tokenization or sentence_split_only
tsv, urls, contexts = read_tsv(tsv_in_file)
tsv.loc[tsv.TOKEN.isnull(), 'TOKEN'] = ""
print("Input file: {}".format(tsv_in_file))
if show_urls:
print("URLS:{}".format(urls))
if tsv['No.'].max() > 0 and just_zero:
print("File {} already has sentence splitting (--just-zero). Skipping.".format(tsv_in_file))
return
if num_tokens:
print("Number of tokens {}". format(len(tsv)), end=" ")
if sentence_count:
print("Number of sentences {}". format(sum(tsv['No.'] == 0)), end=" ")
if max_sentence_len:
print("Maximum sentence length {}.".format(tsv['No.'].max()), end=" ")
if ner_rest_endpoint is None:
tsv_tmp = tsv
else:
tsv_tmp, _ = ner(tsv, ner_rest_endpoint, keep_tokenization=keep_tokenization)
2024-11-12 17:01:34 +01:00
if tsv_out_file is None:
print("\n")
return
print("\n==>")
print("Output file: {}".format(tsv_out_file))
if num_tokens:
print("Number of tokens {}". format(len(tsv_tmp)), end=" ")
if sentence_count:
print("Number of sentences {}". format(sum(tsv_tmp['No.'] == 0)), end=" ")
if max_sentence_len:
print("Maximum sentence length {}.".format(tsv_tmp['No.'].max()), end=" ")
num_diff = -1
if keep_tokenization:
num_diff = sum(tsv.TOKEN != tsv_tmp.TOKEN)
if keep_tokenization and num_diff > 0:
print("Number of token differences: {}".format(num_diff))
raise AssertionError()
# diff = pd.concat([tsv.loc[tsv.TOKEN != tsv_tmp.TOKEN],
# tsv_tmp[['TOKEN']].loc[tsv.TOKEN != tsv_tmp.TOKEN].
# rename(columns={'TOKEN': 'TOKEN_TMP'})], axis=1)
#
# import ipdb;ipdb.set_trace()
if sentence_split_only:
tsv_out = tsv
tsv_out['No.'] = tsv_tmp['No.']
else:
tsv_out = tsv_tmp
if sanitize_sentence_numbers:
word_pos = 0
prev_pos = 0
for idx, _ in tsv_out.iterrows():
2024-11-12 15:15:52 +01:00
if prev_pos != 0 and not str(tsv_out.loc[idx, 'NE-TAG']).startswith('I-') and \
tsv_out.loc[idx, 'No.'] == 0 or len(tsv_out.loc[idx, 'TOKEN']) == 0:
word_pos = 0
prev_pos = word_pos
tsv_out.loc[idx, 'No.'] = word_pos
word_pos += 1
write_tsv(tsv_out, urls, contexts, tsv_out_file)
print("\n")
2021-05-07 12:36:42 +02:00
def page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority, normalization_file):
print("page2tsv - processing file: {}".format(page_xml_file))
2021-02-01 16:25:12 +01:00
if purpose == "NERD":
out_columns = ['No.', 'TOKEN', 'NE-TAG', 'NE-EMB', 'ID', 'url_id', 'left', 'right', 'top', 'bottom', 'conf']
2021-02-01 16:25:12 +01:00
elif purpose == "OCR":
2021-04-01 17:12:58 +02:00
out_columns = ['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'conf', 'line_id']
if min_confidence is not None and max_confidence is not None:
out_columns += ['ocrconf']
2021-02-01 16:25:12 +01:00
else:
raise RuntimeError("Unknown purpose.")
if noproxy:
os.environ['no_proxy'] = '*'
urls = []
if os.path.exists(tsv_out_file):
parts = extract_doc_links(tsv_out_file)
urls = [part['url'] for part in parts]
else:
pd.DataFrame([], columns=out_columns).to_csv(tsv_out_file, sep="\t", quoting=3, index=False)
2021-02-01 16:25:12 +01:00
pcgts = parse(page_xml_file)
2021-02-01 16:25:12 +01:00
tsv = []
line_info = []
_unicode_normalize = unicode_normalize
if normalization_file is not None:
normalization_map = pd.read_pickle(normalization_file)
normalization_map = normalization_map.set_index('decimal')
_unicode_normalize = lambda s: unicode_normalize(s, normalization_map=normalization_map)
for region_idx, region in enumerate(pcgts.get_Page().get_AllRegions(classes=['Text'], order='reading-order')):
for text_line in region.get_TextLine():
2021-05-07 11:28:24 +02:00
left, top, right, bottom = [int(scale_factor * x) for x in bbox_from_points(text_line.get_Coords().points)]
2021-02-01 16:25:12 +01:00
if min_confidence is not None and max_confidence is not None:
conf = np.max([textequiv.conf for textequiv in text_line.get_TextEquiv()])
else:
conf = np.nan
2021-04-01 17:12:58 +02:00
line_info.append((len(urls), left, right, top, bottom, conf, text_line.id))
2021-02-01 16:25:12 +01:00
2021-11-19 11:27:46 +01:00
words = [word for word in text_line.get_Word()]
if len(words) <= 0:
for text_equiv in text_line.get_TextEquiv():
# transform OCR coordinates using `scale_factor` to derive
# correct coordinates for the web presentation image
2022-02-21 15:22:04 +01:00
left, top, right, bottom = [int(scale_factor * x) for x in bbox_from_points(text_line.get_Coords().points)]
text = text_equiv.get_Unicode()
for text_part in text.split(" "):
tsv.append((region_idx, len(line_info) - 1, left + (right - left) / 2.0,
_unicode_normalize(text_part), len(urls), left, right, top, bottom, text_line.id))
2021-11-19 11:27:46 +01:00
else:
for word in words:
# XXX TODO make this configurable
textequiv = ''
list_textequivs = word.get_TextEquiv()
if list_textequivs:
textequiv = list_textequivs[0].get_Unicode()
# transform OCR coordinates using `scale_factor` to derive
# correct coordinates for the web presentation image
left, top, right, bottom = [int(scale_factor * x) for x in bbox_from_points(word.get_Coords().points)]
tsv.append((region_idx, len(line_info) - 1, left + (right - left) / 2.0,
_unicode_normalize(textequiv), len(urls), left, right, top, bottom, text_line.id))
2021-02-01 16:25:12 +01:00
2021-04-01 17:12:58 +02:00
line_info = pd.DataFrame(line_info, columns=['url_id', 'left', 'right', 'top', 'bottom', 'conf', 'line_id'])
if min_confidence is not None and max_confidence is not None:
line_info['ocrconf'] = line_info.conf.map(lambda x: get_conf_color(x, min_confidence, max_confidence))
2021-02-01 16:25:12 +01:00
2021-05-06 16:23:08 +02:00
tsv = pd.DataFrame(tsv, columns=['rid', 'line', 'hcenter'] +
['TEXT', 'url_id', 'left', 'right', 'top', 'bottom', 'line_id'])
2021-02-01 16:25:12 +01:00
2022-02-21 15:22:04 +01:00
# print(tsv)
2021-02-01 16:25:12 +01:00
with open(tsv_out_file, 'a') as f:
f.write('# ' + image_url + '\n')
2022-02-21 15:22:04 +01:00
if len(tsv) == 0:
return
2021-02-01 16:25:12 +01:00
vlinecenter = pd.DataFrame(tsv[['line', 'top']].groupby('line', sort=False).mean().top +
(tsv[['line', 'bottom']].groupby('line', sort=False).mean().bottom -
tsv[['line', 'top']].groupby('line', sort=False).mean().top) / 2,
columns=['vlinecenter'])
tsv = tsv.merge(vlinecenter, left_on='line', right_index=True)
regions = [region.sort_values(['vlinecenter', 'hcenter']) for rid, region in tsv.groupby('rid', sort=False)]
tsv = pd.concat(regions)
if purpose == 'NERD':
tsv['No.'] = 0
tsv['NE-TAG'] = 'O'
tsv['NE-EMB'] = 'O'
tsv['ID'] = '-'
tsv['conf'] = '-'
2021-02-01 16:25:12 +01:00
tsv = tsv.rename(columns={'TEXT': 'TOKEN'})
2022-02-21 15:22:04 +01:00
elif purpose == 'OCR':
2021-02-01 16:25:12 +01:00
tsv = pd.DataFrame([(line, " ".join(part.TEXT.to_list())) for line, part in tsv.groupby('line')],
columns=['line', 'TEXT'])
2021-03-10 15:33:30 +01:00
tsv = tsv.merge(line_info, left_on='line', right_index=True)
2021-02-01 16:25:12 +01:00
tsv = tsv[out_columns].reset_index(drop=True)
try:
if purpose == 'NERD' and ner_rest_endpoint is not None:
tsv, ner_result = ner(tsv, ner_rest_endpoint)
if ned_rest_endpoint is not None:
2021-05-06 16:23:08 +02:00
tsv, _ = ned(tsv, ner_result, ned_rest_endpoint, threshold=ned_threshold, priority=ned_priority)
tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False, encoding='utf-8')
2021-02-01 16:25:12 +01:00
except requests.HTTPError as e:
print(e)
2023-10-20 16:36:43 +02:00
def tsv2page(output_filename, keep_words, page_file, tsv_file):
if not output_filename:
output_filename = Path(page_file).stem + '.corrected.xml'
ns = {'pc': 'http://schema.primaresearch.org/PAGE/gts/pagecontent/2019-07-15'}
tsv = pd.read_csv(tsv_file, sep='\t', comment='#', quoting=3)
tree = ET.parse(page_file)
for _, row in tsv.iterrows():
el_textline = tree.find(f'//pc:TextLine[@id="{row.line_id}"]', namespaces=ns)
el_textline.find('pc:TextEquiv/pc:Unicode', namespaces=ns).text = row.TEXT
if not keep_words:
for el_word in el_textline.findall('pc:Word', namespaces=ns):
el_textline.remove(el_word)
with open(output_filename, 'w', encoding='utf-8') as f:
f.write(ET.tostring(tree, pretty_print=True).decode('utf-8'))
2021-02-01 16:25:12 +01:00
2023-10-20 16:36:43 +02:00
@click.command()
@click.option('--output-filename', '-o', help="Output filename. "
"If omitted, PAGE-XML filename with .corrected.xml extension")
@click.option('--keep-words', '-k', is_flag=True, help="Keep (out-of-date) Words of TextLines")
@click.argument('page-file')
@click.argument('tsv-file')
def tsv2page_cli(output_filename, keep_words, page_file, tsv_file):
return tsv2page(output_filename, keep_words, page_file, tsv_file)
2021-05-06 12:24:47 +02:00
2023-10-20 16:36:43 +02:00
2021-02-01 16:25:12 +01:00
@click.command()
@click.option('--xls-file', type=click.Path(exists=True), default=None,
help="Read parameters from xls-file. Expected columns: Filename, iiif_url, scale_factor.")
@click.option('--directory', type=click.Path(exists=True), default=None,
help="Search directory for PPN**/*.xml files. Extract PPN and file number into image-url.")
2021-02-01 16:25:12 +01:00
@click.option('--purpose', type=click.Choice(['NERD', 'OCR'], case_sensitive=False), default="NERD",
help="Purpose of output tsv file. "
"\n\nNERD: NER/NED application/ground-truth creation. "
"\n\nOCR: OCR application/ground-truth creation. "
"\n\ndefault: NERD.")
def make_page2tsv_commands(xls_file, directory, purpose):
if xls_file is not None:
if xls_file.endswith(".xls"):
df = pd.read_excel(xls_file)
else:
df = pd.read_excel(xls_file, engine='openpyxl')
2021-02-01 16:25:12 +01:00
df = df.dropna(how='all')
for _, row in df.iterrows():
print('page2tsv $(OPTIONS) {}.xml {}.tsv --image-url={} --scale-factor={} --purpose={}'.
format(row.Filename, row.Filename, row.iiif_url.replace('/full/full', '/left,top,width,height/full'),
row.scale_factor, purpose))
elif directory is not None:
for file in glob.glob('{}/**/*.xml'.format(directory), recursive=True):
2021-04-09 08:07:27 +02:00
ma = re.match('(.*/(PPN[0-9X]+)/.*?([0-9]+).*?).xml', file)
if ma:
print('page2tsv {} {}.tsv '
'--image-url=https://content.staatsbibliothek-berlin.de/dc/'
'{}-{:08d}/left,top,width,height/full/0/default.jpg --scale-factor=1.0 --purpose={}'.
2021-03-10 14:06:51 +01:00
format(file, ma.group(1), ma.group(2), int(ma.group(3)), purpose))
2021-02-01 16:25:12 +01:00
2022-02-21 15:22:04 +01:00
@click.command()
@click.argument('page-xml-file', type=click.Path(exists=True), required=True, nargs=1)
@click.argument('tsv-out-file', type=click.Path(), required=True, nargs=1)
@click.option('--purpose', type=click.Choice(['NERD', 'OCR'], case_sensitive=False), default="NERD",
help="Purpose of output tsv file. "
"\n\nNERD: NER/NED application/ground-truth creation. "
"\n\nOCR: OCR application/ground-truth creation. "
"\n\ndefault: NERD.")
@click.option('--image-url', type=str, default='http://empty')
@click.option('--ner-rest-endpoint', type=str, default=None,
help="REST endpoint of sbb_ner service. See https://github.com/qurator-spk/sbb_ner for details. "
"Only applicable in case of NERD.")
@click.option('--ned-rest-endpoint', type=str, default=None,
help="REST endpoint of sbb_ned service. See https://github.com/qurator-spk/sbb_ned for details. "
"Only applicable in case of NERD.")
@click.option('--noproxy', type=bool, is_flag=True, help='disable proxy. default: enabled.')
@click.option('--scale-factor', type=float, default=1.0, help='default: 1.0')
@click.option('--ned-threshold', type=float, default=None)
@click.option('--min-confidence', type=float, default=None)
@click.option('--max-confidence', type=float, default=None)
@click.option('--ned-priority', type=int, default=1)
@click.option('--normalization-file', type=click.Path(exists=True), default=None)
2022-02-21 15:22:04 +01:00
def page2tsv_cli(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority, normalization_file):
2022-02-21 15:22:04 +01:00
return page2tsv(page_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
noproxy, scale_factor, ned_threshold, min_confidence, max_confidence, ned_priority, normalization_file)
2023-10-20 16:36:43 +02:00
@click.command()
@click.argument('alto-xml-file', type=click.Path(exists=True), required=True, nargs=1)
@click.argument('tsv-out-file', type=click.Path(), required=True, nargs=1)
@click.option('--purpose', type=click.Choice(['NERD', 'OCR'], case_sensitive=False), default="NERD",
help="Purpose of output tsv file. "
"\n\nNERD: NER/NED application/ground-truth creation. "
"\n\nOCR: OCR application/ground-truth creation. "
"\n\ndefault: NERD.")
@click.option('--image-url', type=str, default='http://empty')
@click.option('--ner-rest-endpoint', type=str, default=None,
help="REST endpoint of sbb_ner service. See https://github.com/qurator-spk/sbb_ner for details. "
"Only applicable in case of NERD.")
@click.option('--ned-rest-endpoint', type=str, default=None,
help="REST endpoint of sbb_ned service. See https://github.com/qurator-spk/sbb_ned for details. "
"Only applicable in case of NERD.")
@click.option('--noproxy', type=bool, is_flag=True, help='disable proxy. default: enabled.')
@click.option('--scale-factor', type=float, default=1.0, help='default: 1.0')
@click.option('--ned-threshold', type=float, default=None)
@click.option('--ned-priority', type=int, default=1)
def alto2tsv_cli(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
noproxy, scale_factor, ned_threshold, ned_priority):
return alto2tsv(alto_xml_file, tsv_out_file, purpose, image_url, ner_rest_endpoint, ned_rest_endpoint,
noproxy, scale_factor, ned_threshold, ned_priority)