add annotation tools and url mapping integration

tools/cli.py

@@ -0,0 +1,97 @@
+import re
+import click
+import pandas as pd
+from io import StringIO
+
+
+@click.command()
+@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
+def extract_document_links(tsv_file):
+
+    parts = extract_doc_links(tsv_file)
+
+    for part in parts:
+
+        print(part['url'])
+
+
+@click.command()
+@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
+@click.argument('annotated-tsv-file', type=click.Path(exists=False), required=True, nargs=1)
+def annotate_tsv(tsv_file, annotated_tsv_file):
+
+    parts = extract_doc_links(tsv_file)
+
+    annotated_parts = []
+
+    urls = []
+
+    for part in parts:
+
+        part_data = StringIO(part['header'] + part['text'])
+        urls.append(part['url'])
+
+        df = pd.read_csv(part_data, sep="\t", comment='#', quoting=3)
+
+        df['url_id'] = len(annotated_parts)
+
+        annotated_parts.append(df)
+
+    df = pd.concat(annotated_parts)
+
+    df.to_csv(annotated_tsv_file, sep="\t", quoting=3, index=False)
+
+
+def extract_doc_links(tsv_file):
+
+    parts = []
+
+    header = None
+
+    with open(tsv_file, 'r') as f:
+        
+        text = []
+        url = None
+
+        for line in f:
+
+            if header is None:
+                header = "\t".join(line.split()) + '\n'
+                continue
+
+            urls = [url for url in
+                    re.findall(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', line)]
+
+            if len(urls) > 0:
+                if url is not None:
+                    parts.append({"url": url, 'header': header, 'text': "".join(text)})
+                    text = []
+
+                url = urls[-1]
+            else:
+                if url is None:
+                    continue
+
+                line = '\t'.join(line.split())
+
+                if line.count('\t') == 2:
+
+                    line = "\t" + line
+
+                if line.count('\t') == 3:
+
+                    text.append(line + '\n')
+
+                    continue
+
+                if line.startswith('#'):
+                    continue
+
+                if len(line) == 0:
+                    continue
+
+                print('Line error: |', line, '|Number of Tabs: ', line.count('\t'))
+
+        parts.append({"url": url, 'header': header, 'text': "".join(text)})
+
+    return parts

tools/requirements.txt

@@ -0,0 +1,3 @@
+numpy
+pandas
+click

tools/setup.py

@@ -0,0 +1,35 @@
+from io import open
+from setuptools import find_packages, setup
+
+with open('requirements.txt') as fp:
+    install_requires = fp.read()
+
+setup(
+    name="ner-edith",
+    version="0.0.1",
+    author="",
+    author_email="qurator@sbb.spk-berlin.de",
+    description="ner.edith",
+    long_description=open("README.md", "r", encoding='utf-8').read(),
+    long_description_content_type="text/markdown",
+    keywords='qurator',
+    license='Apache',
+    url="https://github.com/cneud/ner.edith",
+    packages=find_packages(exclude=["*.tests", "*.tests.*",
+                                    "tests.*", "tests"]),
+    install_requires=install_requires,
+    entry_points={
+      'console_scripts': [
+        "extract-doc-links=cli:extract_document_links",
+        "annotate-tsv=cli:annotate_tsv"
+      ]
+    },
+    python_requires='>=3.6.0',
+    tests_require=['pytest'],
+    classifiers=[
+          'Intended Audience :: Science/Research',
+          'License :: OSI Approved :: Apache Software License',
+          'Programming Language :: Python :: 3',
+          'Topic :: Scientific/Engineering :: Artificial Intelligence',
+    ],
+)