1
0
Fork 0
mirror of https://github.com/qurator-spk/neat.git synced 2025-06-09 11:49:54 +02:00

add annotation tools and url mapping integration

This commit is contained in:
Kai Labusch 2019-09-23 16:31:02 +02:00
parent 3d10f7b001
commit 6afb0a6375
5 changed files with 345 additions and 155 deletions

0
tools/README.md Normal file
View file

97
tools/cli.py Normal file
View file

@ -0,0 +1,97 @@
import re
import click
import pandas as pd
from io import StringIO
@click.command()
@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
def extract_document_links(tsv_file):
parts = extract_doc_links(tsv_file)
for part in parts:
print(part['url'])
@click.command()
@click.argument('tsv-file', type=click.Path(exists=True), required=True, nargs=1)
@click.argument('annotated-tsv-file', type=click.Path(exists=False), required=True, nargs=1)
def annotate_tsv(tsv_file, annotated_tsv_file):
parts = extract_doc_links(tsv_file)
annotated_parts = []
urls = []
for part in parts:
part_data = StringIO(part['header'] + part['text'])
urls.append(part['url'])
df = pd.read_csv(part_data, sep="\t", comment='#', quoting=3)
df['url_id'] = len(annotated_parts)
annotated_parts.append(df)
df = pd.concat(annotated_parts)
df.to_csv(annotated_tsv_file, sep="\t", quoting=3, index=False)
def extract_doc_links(tsv_file):
parts = []
header = None
with open(tsv_file, 'r') as f:
text = []
url = None
for line in f:
if header is None:
header = "\t".join(line.split()) + '\n'
continue
urls = [url for url in
re.findall(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', line)]
if len(urls) > 0:
if url is not None:
parts.append({"url": url, 'header': header, 'text': "".join(text)})
text = []
url = urls[-1]
else:
if url is None:
continue
line = '\t'.join(line.split())
if line.count('\t') == 2:
line = "\t" + line
if line.count('\t') == 3:
text.append(line + '\n')
continue
if line.startswith('#'):
continue
if len(line) == 0:
continue
print('Line error: |', line, '|Number of Tabs: ', line.count('\t'))
parts.append({"url": url, 'header': header, 'text': "".join(text)})
return parts

3
tools/requirements.txt Normal file
View file

@ -0,0 +1,3 @@
numpy
pandas
click

35
tools/setup.py Normal file
View file

@ -0,0 +1,35 @@
from io import open
from setuptools import find_packages, setup
with open('requirements.txt') as fp:
install_requires = fp.read()
setup(
name="ner-edith",
version="0.0.1",
author="",
author_email="qurator@sbb.spk-berlin.de",
description="ner.edith",
long_description=open("README.md", "r", encoding='utf-8').read(),
long_description_content_type="text/markdown",
keywords='qurator',
license='Apache',
url="https://github.com/cneud/ner.edith",
packages=find_packages(exclude=["*.tests", "*.tests.*",
"tests.*", "tests"]),
install_requires=install_requires,
entry_points={
'console_scripts': [
"extract-doc-links=cli:extract_document_links",
"annotate-tsv=cli:annotate_tsv"
]
},
python_requires='>=3.6.0',
tests_require=['pytest'],
classifiers=[
'Intended Audience :: Science/Research',
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python :: 3',
'Topic :: Scientific/Engineering :: Artificial Intelligence',
],
)