qurator-spk/dinglehopper
1
0
Fork 0
mirror of https://github.com/qurator-spk/dinglehopper.git synced 2025-06-20 00:59:58 +02:00
Code Issues Releases Wiki Activity

🚧 dinglehopper: Test aligning by character while retaining segment id info

Browse source
This commit is contained in:
Gerber, Mike 2020-06-11 13:54:46 +02:00
parent 4cd835ae51
commit 167dad18f4
1 changed files with 19 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

24
extracted_text_test.py
View file

@ -3,6 +3,7 @@ import pytest
from extracted_text import ExtractedText, ExtractedTextSegment
from uniseg.graphemecluster import grapheme_clusters
from qurator.dinglehopper import seq_align
from collections import namedtuple
def test_text():
@ -24,6 +25,9 @@ def test_normalization_check():
assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ'))
AlignmentElement = namedtuple('AlignmentElement', 'left right left_id right_id')
def test_align():
"""
Test aligning by character while retaining segment id info
@ -35,20 +39,30 @@ def test_align():
test1 = ExtractedText([
ExtractedTextSegment('s0', 'foo'),
ExtractedTextSegment('s1', 'bar'),
ExtractedTextSegment('s2', 'bazinga')
ExtractedTextSegment('s2', 'batzinga')
], ' ')
test2 = ExtractedText([
ExtractedTextSegment('x0', 'foo'),
ExtractedTextSegment('x1', 'bar'),
ExtractedTextSegment('x2', '.'), # extra .
ExtractedTextSegment('x2', 'bazim̃ga'), # different grapheme cluster, m̃ also is two Python characters
ExtractedTextSegment('x3', 'bazim̃ga'), # deletion + different grapheme cluster, m̃ also is two Python characters
], ' ')
left_pos = 0; right_pos = 0
left_pos = 0; right_pos = 0; alignment = []
for left, right in seq_align(grapheme_clusters(test1.text), grapheme_clusters(test2.text)):
print(left, right, test1.segment_id_for_pos(left_pos), test2.segment_id_for_pos(right_pos))
left_id = test1.segment_id_for_pos(left_pos) if left is not None else None
right_id = test2.segment_id_for_pos(right_pos) if right is not None else None
el = AlignmentElement(left, right, left_id, right_id)
alignment.append(el)
if left is not None:
left_pos += len(left)
if right is not None:
right_pos += len(right)
assert False
print('test1: {}'.format(test1.text))
print('test2: {}'.format(test2.text))
assert alignment[0] == ('f', 'f', 's0', 'x0')
assert alignment[8] == (None, '.', None, 'x2')
assert alignment[12] == ('t', None, 's2', None)
assert alignment[15] == ('n', 'm̃', 's2', 'x3')