qurator-spk/dinglehopper
1
0
Fork 0
mirror of https://github.com/qurator-spk/dinglehopper.git synced 2025-07-11 03:19:57 +02:00
Code Issues Releases Wiki Activity

🚧 dinglehopper: WIP data structure for extracted text

Browse source
This commit is contained in:
Gerber, Mike 2020-06-10 19:36:49 +02:00
parent 2c69e077fe
commit bc630233d0
1 changed files with 19 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

19
extracted_text.py
View file

@ -1,4 +1,5 @@
import attr import attr
import unicodedata
@attr.s(frozen=True) @attr.s(frozen=True)
@ -23,10 +24,25 @@ class ExtractedText:
i += len(self.joiner) i += len(self.joiner)
NORM_NFC = 0
def normalize(text, normalization):
if normalization == NORM_NFC:
return unicodedata.normalize('NFC', text)
else:
raise ValueError()
@attr.s(frozen=True) @attr.s(frozen=True)
class ExtractedTextSegment: class ExtractedTextSegment:
id = attr.ib(type=str) id = attr.ib(type=str)
text = attr.ib(type=str) text = attr.ib(type=str)
@text.validator
def check(self, attribute, value):
if normalize(value, self.normalization) != value:
raise ValueError('String "{}" is not normalized.'.format(value))
normalization = attr.ib(default=NORM_NFC)
test1 = ExtractedText([ test1 = ExtractedText([
@ -40,3 +56,6 @@ assert test1.text == 'foo bar bazinga'
assert test1.segment_id_for_pos(0) == 's0' assert test1.segment_id_for_pos(0) == 's0'
assert test1.segment_id_for_pos(3) == None assert test1.segment_id_for_pos(3) == None
assert test1.segment_id_for_pos(10) == 's2' assert test1.segment_id_for_pos(10) == 's2'
# ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ'))
ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ'))