🚧 dinglehopper: Test aligning by character while retaining segment id info

pull/38/head
Gerber, Mike 4 years ago
parent 354afdc0b2
commit 1d553bb4e3

@ -1,6 +1,8 @@
import unicodedata import unicodedata
import pytest import pytest
from extracted_text import ExtractedText, ExtractedTextSegment from extracted_text import ExtractedText, ExtractedTextSegment
from uniseg.graphemecluster import grapheme_clusters
from qurator.dinglehopper import seq_align
def test_text(): def test_text():
@ -20,3 +22,33 @@ def test_normalization_check():
with pytest.raises(ValueError, match=r'.*is not normalized.*'): with pytest.raises(ValueError, match=r'.*is not normalized.*'):
ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ')) ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ'))
assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ')) assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ'))
def test_align():
"""
Test aligning by character while retaining segment id info
The difficulty here is that aligning should work on grapheme clusters,
not Python characters.
"""
test1 = ExtractedText([
ExtractedTextSegment('s0', 'foo'),
ExtractedTextSegment('s1', 'bar'),
ExtractedTextSegment('s2', 'bazinga')
], ' ')
test2 = ExtractedText([
ExtractedTextSegment('x0', 'foo'),
ExtractedTextSegment('x1', 'bar'),
ExtractedTextSegment('x2', '.'), # extra .
ExtractedTextSegment('x2', 'bazim̃ga'), # different grapheme cluster, m̃ also is two Python characters
], ' ')
left_pos = 0; right_pos = 0
for left, right in seq_align(grapheme_clusters(test1.text), grapheme_clusters(test2.text)):
print(left, right, test1.segment_id_for_pos(left_pos), test2.segment_id_for_pos(right_pos))
if left is not None:
left_pos += len(left)
if right is not None:
right_pos += len(right)
assert False

Loading…
Cancel
Save