qurator-spk/page2tsv
1
0
Fork 0
mirror of https://github.com/qurator-spk/page2tsv.git synced 2025-07-26 10:29:54 +02:00
Code Issues Releases Wiki Activity

add findentities command line tool that can be used in order to NER/NED tag an existing .tsv file

Browse source
This commit is contained in:
Kai 2020-03-25 08:18:59 +01:00
parent b13dae29f5
commit 24fd7245f5
2 changed files with 41 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

44
cli.py
View file

@ -107,6 +107,8 @@ def ner(tsv, ner_rest_endpoint):
resp = requests.post(url=ner_rest_endpoint, json={'text': " ".join(tsv.TOKEN.tolist())}) resp = requests.post(url=ner_rest_endpoint, json={'text': " ".join(tsv.TOKEN.tolist())})
resp.raise_for_status()
def iterate_ner_results(result_sentences): def iterate_ner_results(result_sentences):
for sen in result_sentences: for sen in result_sentences:
@ -148,10 +150,14 @@ def ned(tsv, ner_result, ned_rest_endpoint):
resp = requests.post(url=ned_rest_endpoint + '/parse', json=ner_result) resp = requests.post(url=ned_rest_endpoint + '/parse', json=ner_result)
resp.raise_for_status()
ner_parsed = json.loads(resp.content) ner_parsed = json.loads(resp.content)
resp = requests.post(url=ned_rest_endpoint + '/ned', json=ner_parsed, timeout=3600000) resp = requests.post(url=ned_rest_endpoint + '/ned', json=ner_parsed, timeout=3600000)
resp.raise_for_status()
ned_result = json.loads(resp.content) ned_result = json.loads(resp.content)
rids = [] rids = []
@ -261,12 +267,40 @@ def page2tsv(page_xml_file, tsv_out_file, image_url, ner_rest_endpoint, ned_rest
tsv = tsv[out_columns].reset_index(drop=True) tsv = tsv[out_columns].reset_index(drop=True)
if ner_rest_endpoint is not None: try:
if ner_rest_endpoint is not None:
tsv, ner_result = ner(tsv, ner_rest_endpoint) tsv, ner_result = ner(tsv, ner_rest_endpoint)
if ned_rest_endpoint is not None: if ned_rest_endpoint is not None:
tsv = ned(tsv, ner_result, ned_rest_endpoint) tsv = ned(tsv, ner_result, ned_rest_endpoint)
tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False) tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
except requests.HTTPError as e:
print(e)
@click.command()
@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
@click.argument('tsv-out-file', type=click.Path(), required=True, nargs=1)
@click.option('--ner-rest-endpoint', type=str, default=None,
help="REST endpoint of sbb_ner service. See https://github.com/qurator-spk/sbb_ner for details.")
@click.option('--ned-rest-endpoint', type=str, default=None,
help="REST endpoint of sbb_ned service. See https://github.com/qurator-spk/sbb_ned for details.")
def find_entities(tsv_file, tsv_out_file, ner_rest_endpoint, ned_rest_endpoint):
tsv = pd.read_csv(tsv_file, sep='\t', comment='#', quoting=3)
try:
if ner_rest_endpoint is not None:
tsv, ner_result = ner(tsv, ner_rest_endpoint)
if ned_rest_endpoint is not None:
tsv = ned(tsv, ner_result, ned_rest_endpoint)
tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
except requests.HTTPError as e:
print(e)

3
setup.py
View file

@ -22,7 +22,8 @@ setup(
'console_scripts': [ 'console_scripts': [
"extract-doc-links=cli:extract_document_links", "extract-doc-links=cli:extract_document_links",
"annotate-tsv=cli:annotate_tsv", "annotate-tsv=cli:annotate_tsv",
"page2tsv=cli:page2tsv" "page2tsv=cli:page2tsv",
"find-entities=cli:find_entities"
] ]
}, },
python_requires='>=3.6.0', python_requires='>=3.6.0',