add findentities command line tool that can be used in order to NER/NED tag an existing .tsv file

cli.py

@@ -107,6 +107,8 @@ def ner(tsv, ner_rest_endpoint):
 
     resp = requests.post(url=ner_rest_endpoint, json={'text': " ".join(tsv.TOKEN.tolist())})
 
+    resp.raise_for_status()
+
     def iterate_ner_results(result_sentences):
 
         for sen in result_sentences:
@@ -148,10 +150,14 @@ def ned(tsv, ner_result, ned_rest_endpoint):
 
     resp = requests.post(url=ned_rest_endpoint + '/parse', json=ner_result)
 
+    resp.raise_for_status()
+
     ner_parsed = json.loads(resp.content)
 
     resp = requests.post(url=ned_rest_endpoint + '/ned', json=ner_parsed, timeout=3600000)
 
+    resp.raise_for_status()
+
     ned_result = json.loads(resp.content)
 
     rids = []
@@ -261,6 +267,32 @@ def page2tsv(page_xml_file, tsv_out_file, image_url, ner_rest_endpoint, ned_rest
 
     tsv = tsv[out_columns].reset_index(drop=True)
 
+    try:
+        if ner_rest_endpoint is not None:
+
+            tsv, ner_result = ner(tsv, ner_rest_endpoint)
+
+            if ned_rest_endpoint is not None:
+
+                tsv = ned(tsv, ner_result, ned_rest_endpoint)
+
+        tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
+    except requests.HTTPError as e:
+        print(e)
+
+
+@click.command()
+@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
+@click.argument('tsv-out-file', type=click.Path(), required=True, nargs=1)
+@click.option('--ner-rest-endpoint', type=str, default=None,
+              help="REST endpoint of sbb_ner service. See https://github.com/qurator-spk/sbb_ner for details.")
+@click.option('--ned-rest-endpoint', type=str, default=None,
+              help="REST endpoint of sbb_ned service. See https://github.com/qurator-spk/sbb_ned for details.")
+def find_entities(tsv_file, tsv_out_file, ner_rest_endpoint, ned_rest_endpoint):
+
+    tsv = pd.read_csv(tsv_file, sep='\t', comment='#', quoting=3)
+
+    try:
         if ner_rest_endpoint is not None:
 
             tsv, ner_result = ner(tsv, ner_rest_endpoint)
@@ -270,3 +302,5 @@ def page2tsv(page_xml_file, tsv_out_file, image_url, ner_rest_endpoint, ned_rest
                 tsv = ned(tsv, ner_result, ned_rest_endpoint)
 
         tsv.to_csv(tsv_out_file, sep="\t", quoting=3, index=False, mode='a', header=False)
+    except requests.HTTPError as e:
+        print(e)

setup.py

@@ -22,7 +22,8 @@ setup(
       'console_scripts': [
         "extract-doc-links=cli:extract_document_links",
         "annotate-tsv=cli:annotate_tsv",
-        "page2tsv=cli:page2tsv"
+        "page2tsv=cli:page2tsv",
+        "find-entities=cli:find_entities"
       ]
     },
     python_requires='>=3.6.0',