qurator-spk/dinglehopper
1
0
Fork 0
mirror of https://github.com/qurator-spk/dinglehopper.git synced 2025-07-27 19:29:55 +02:00
Code Issues Releases Wiki Activity

🚧 dinglehopper: WIP data structure for extracted text

Browse source
This commit is contained in:
Gerber, Mike 2020-06-10 20:29:01 +02:00
parent 8e3a19d7e9
commit 91371971eb
2 changed files with 16 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

9
extracted_text.py
View file

@ -1,5 +1,6 @@
import attr import attr
import unicodedata import unicodedata
import enum
# TODO handle grapheme cluster positions? # TODO handle grapheme cluster positions?
@ -27,11 +28,13 @@ class ExtractedText:
i += len(self.joiner) i += len(self.joiner)
NORM_NFC = 0 class Normalization(enum.Enum):
NFC = 1
NFC_MUFI = 2
def normalize(text, normalization): def normalize(text, normalization):
if normalization == NORM_NFC: if normalization == Normalization.NFC:
return unicodedata.normalize('NFC', text) return unicodedata.normalize('NFC', text)
else: else:
raise ValueError() raise ValueError()
@ -45,4 +48,4 @@ class ExtractedTextSegment:
def check(self, attribute, value): def check(self, attribute, value):
if normalize(value, self.normalization) != value: if normalize(value, self.normalization) != value:
raise ValueError('String "{}" is not normalized.'.format(value)) raise ValueError('String "{}" is not normalized.'.format(value))
normalization = attr.ib(default=NORM_NFC) normalization = attr.ib(converter=Normalization, default=Normalization.NFC)

15
extracted_text_test.py
View file

@ -1,17 +1,22 @@
from extracted_text import * import unicodedata
import pytest
from extracted_text import ExtractedText, ExtractedTextSegment
def test_text(): def test_text():
test1 = ExtractedText([ test1 = ExtractedText([
ExtractedTextSegment('s0', 'foo'), ExtractedTextSegment('s0', 'foo'),
ExtractedTextSegment('s1', 'bar'), ExtractedTextSegment(1, 'bar'),
ExtractedTextSegment('s2', 'bazinga') ExtractedTextSegment('s2', 'bazinga')
], ' ') ], ' ')
assert test1.text == 'foo bar bazinga' assert test1.text == 'foo bar bazinga'
assert test1.segment_id_for_pos(0) == 's0' assert test1.segment_id_for_pos(0) == 's0'
assert test1.segment_id_for_pos(3) is None assert test1.segment_id_for_pos(3) is None
assert test1.segment_id_for_pos(10) == 's2' assert test1.segment_id_for_pos(10) == 's2'
# ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ'))
ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ')) def test_normalization_check():
with pytest.raises(ValueError, match=r'.*is not normalized.*'):
ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ'))
assert ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ'))