mirror of
https://github.com/qurator-spk/dinglehopper.git
synced 2025-06-09 11:50:00 +02:00
🚧 dinglehopper: Extract text while retaining segment id info
This commit is contained in:
parent
5dbf563d6a
commit
a09c1eae7e
1 changed files with 0 additions and 0 deletions
71
qurator/dinglehopper/tests/extracted_text_test.py
Normal file
71
qurator/dinglehopper/tests/extracted_text_test.py
Normal file
|
@ -0,0 +1,71 @@
|
|||
import unicodedata
|
||||
import pytest
|
||||
from qurator.dinglehopper import ExtractedText, ExtractedTextSegment
|
||||
from uniseg.graphemecluster import grapheme_clusters
|
||||
from qurator.dinglehopper import seq_align
|
||||
from collections import namedtuple
|
||||
|
||||
|
||||
def test_text():
|
||||
test1 = ExtractedText([
|
||||
ExtractedTextSegment('s0', 'foo'),
|
||||
ExtractedTextSegment('s1', 'bar'),
|
||||
ExtractedTextSegment('s2', 'bazinga')
|
||||
], ' ')
|
||||
|
||||
assert test1.text == 'foo bar bazinga'
|
||||
assert test1.segment_id_for_pos(0) == 's0'
|
||||
assert test1.segment_id_for_pos(3) is None
|
||||
assert test1.segment_id_for_pos(10) == 's2'
|
||||
|
||||
|
||||
def test_normalization_check():
|
||||
with pytest.raises(ValueError, match=r'.*is not normalized.*'):
|
||||
ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ'))
|
||||
assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ'))
|
||||
|
||||
|
||||
AlignmentElement = namedtuple('AlignmentElement', 'left right left_id right_id')
|
||||
|
||||
|
||||
def test_align():
|
||||
"""
|
||||
Test aligning by character while retaining segment id info
|
||||
|
||||
The difficulty here is that aligning should work on grapheme clusters,
|
||||
not Python characters.
|
||||
"""
|
||||
|
||||
test1 = ExtractedText([
|
||||
ExtractedTextSegment('s0', 'foo'),
|
||||
ExtractedTextSegment('s1', 'bar'),
|
||||
ExtractedTextSegment('s2', 'batzinga')
|
||||
], ' ')
|
||||
test2 = ExtractedText([
|
||||
ExtractedTextSegment('x0', 'foo'),
|
||||
ExtractedTextSegment('x1', 'bar'),
|
||||
ExtractedTextSegment('x2', '.'), # extra .
|
||||
ExtractedTextSegment('x3', 'bazim̃ga'), # deletion + different grapheme cluster, m̃ also is two Python characters
|
||||
], ' ')
|
||||
|
||||
left_pos = 0; right_pos = 0; alignment = []
|
||||
for left, right in seq_align(grapheme_clusters(test1.text), grapheme_clusters(test2.text)):
|
||||
left_id = test1.segment_id_for_pos(left_pos) if left is not None else None
|
||||
right_id = test2.segment_id_for_pos(right_pos) if right is not None else None
|
||||
# XXX note that deletions and inserts only produce one id + None, UI must
|
||||
# support this, i.e. display for the one id produced
|
||||
# XXX otherwise, it should always display for BOTH ids
|
||||
el = AlignmentElement(left, right, left_id, right_id)
|
||||
alignment.append(el)
|
||||
if left is not None:
|
||||
left_pos += len(left)
|
||||
if right is not None:
|
||||
right_pos += len(right)
|
||||
|
||||
print('test1: {}'.format(test1.text))
|
||||
print('test2: {}'.format(test2.text))
|
||||
|
||||
assert alignment[0] == ('f', 'f', 's0', 'x0')
|
||||
assert alignment[8] == (None, '.', None, 'x2')
|
||||
assert alignment[12] == ('t', None, 's2', None)
|
||||
assert alignment[15] == ('n', 'm̃', 's2', 'x3')
|
Loading…
Add table
Add a link
Reference in a new issue