From 167dad18f4e8b6fe48325881fe0c383032dc9b7c Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 11 Jun 2020 13:54:46 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20dinglehopper:=20=20Test=20aligni?= =?UTF-8?q?ng=20by=20character=20while=20retaining=20segment=20id=20info?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extracted_text_test.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/extracted_text_test.py b/extracted_text_test.py index 890b045..60785b7 100644 --- a/extracted_text_test.py +++ b/extracted_text_test.py @@ -3,6 +3,7 @@ import pytest from extracted_text import ExtractedText, ExtractedTextSegment from uniseg.graphemecluster import grapheme_clusters from qurator.dinglehopper import seq_align +from collections import namedtuple def test_text(): @@ -24,6 +25,9 @@ def test_normalization_check(): assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ')) +AlignmentElement = namedtuple('AlignmentElement', 'left right left_id right_id') + + def test_align(): """ Test aligning by character while retaining segment id info @@ -35,20 +39,30 @@ def test_align(): test1 = ExtractedText([ ExtractedTextSegment('s0', 'foo'), ExtractedTextSegment('s1', 'bar'), - ExtractedTextSegment('s2', 'bazinga') + ExtractedTextSegment('s2', 'batzinga') ], ' ') test2 = ExtractedText([ ExtractedTextSegment('x0', 'foo'), ExtractedTextSegment('x1', 'bar'), ExtractedTextSegment('x2', '.'), # extra . - ExtractedTextSegment('x2', 'bazim̃ga'), # different grapheme cluster, m̃ also is two Python characters + ExtractedTextSegment('x3', 'bazim̃ga'), # deletion + different grapheme cluster, m̃ also is two Python characters ], ' ') - left_pos = 0; right_pos = 0 + left_pos = 0; right_pos = 0; alignment = [] for left, right in seq_align(grapheme_clusters(test1.text), grapheme_clusters(test2.text)): - print(left, right, test1.segment_id_for_pos(left_pos), test2.segment_id_for_pos(right_pos)) + left_id = test1.segment_id_for_pos(left_pos) if left is not None else None + right_id = test2.segment_id_for_pos(right_pos) if right is not None else None + el = AlignmentElement(left, right, left_id, right_id) + alignment.append(el) if left is not None: left_pos += len(left) if right is not None: right_pos += len(right) - assert False + + print('test1: {}'.format(test1.text)) + print('test2: {}'.format(test2.text)) + + assert alignment[0] == ('f', 'f', 's0', 'x0') + assert alignment[8] == (None, '.', None, 'x2') + assert alignment[12] == ('t', None, 's2', None) + assert alignment[15] == ('n', 'm̃', 's2', 'x3')