From 1d553bb4e3bf5cb6b8dd1ebb21f576a810d151ef Mon Sep 17 00:00:00 2001 From: "Gerber, Mike" Date: Thu, 11 Jun 2020 13:04:36 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20dinglehopper:=20=20Test=20aligni?= =?UTF-8?q?ng=20by=20character=20while=20retaining=20segment=20id=20info?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extracted_text_test.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/extracted_text_test.py b/extracted_text_test.py index 4919a76..890b045 100644 --- a/extracted_text_test.py +++ b/extracted_text_test.py @@ -1,6 +1,8 @@ import unicodedata import pytest from extracted_text import ExtractedText, ExtractedTextSegment +from uniseg.graphemecluster import grapheme_clusters +from qurator.dinglehopper import seq_align def test_text(): @@ -20,3 +22,33 @@ def test_normalization_check(): with pytest.raises(ValueError, match=r'.*is not normalized.*'): ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ')) assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ')) + + +def test_align(): + """ + Test aligning by character while retaining segment id info + + The difficulty here is that aligning should work on grapheme clusters, + not Python characters. + """ + + test1 = ExtractedText([ + ExtractedTextSegment('s0', 'foo'), + ExtractedTextSegment('s1', 'bar'), + ExtractedTextSegment('s2', 'bazinga') + ], ' ') + test2 = ExtractedText([ + ExtractedTextSegment('x0', 'foo'), + ExtractedTextSegment('x1', 'bar'), + ExtractedTextSegment('x2', '.'), # extra . + ExtractedTextSegment('x2', 'bazim̃ga'), # different grapheme cluster, m̃ also is two Python characters + ], ' ') + + left_pos = 0; right_pos = 0 + for left, right in seq_align(grapheme_clusters(test1.text), grapheme_clusters(test2.text)): + print(left, right, test1.segment_id_for_pos(left_pos), test2.segment_id_for_pos(right_pos)) + if left is not None: + left_pos += len(left) + if right is not None: + right_pos += len(right) + assert False