mirror of
https://github.com/qurator-spk/dinglehopper.git
synced 2025-07-10 19:09:58 +02:00
🚧 dinglehopper: WIP data structure for extracted text
This commit is contained in:
parent
2c69e077fe
commit
bc630233d0
1 changed files with 19 additions and 0 deletions
|
@ -1,4 +1,5 @@
|
||||||
import attr
|
import attr
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
|
||||||
@attr.s(frozen=True)
|
@attr.s(frozen=True)
|
||||||
|
@ -23,10 +24,25 @@ class ExtractedText:
|
||||||
i += len(self.joiner)
|
i += len(self.joiner)
|
||||||
|
|
||||||
|
|
||||||
|
NORM_NFC = 0
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(text, normalization):
|
||||||
|
if normalization == NORM_NFC:
|
||||||
|
return unicodedata.normalize('NFC', text)
|
||||||
|
else:
|
||||||
|
raise ValueError()
|
||||||
|
|
||||||
|
|
||||||
@attr.s(frozen=True)
|
@attr.s(frozen=True)
|
||||||
class ExtractedTextSegment:
|
class ExtractedTextSegment:
|
||||||
id = attr.ib(type=str)
|
id = attr.ib(type=str)
|
||||||
text = attr.ib(type=str)
|
text = attr.ib(type=str)
|
||||||
|
@text.validator
|
||||||
|
def check(self, attribute, value):
|
||||||
|
if normalize(value, self.normalization) != value:
|
||||||
|
raise ValueError('String "{}" is not normalized.'.format(value))
|
||||||
|
normalization = attr.ib(default=NORM_NFC)
|
||||||
|
|
||||||
|
|
||||||
test1 = ExtractedText([
|
test1 = ExtractedText([
|
||||||
|
@ -40,3 +56,6 @@ assert test1.text == 'foo bar bazinga'
|
||||||
assert test1.segment_id_for_pos(0) == 's0'
|
assert test1.segment_id_for_pos(0) == 's0'
|
||||||
assert test1.segment_id_for_pos(3) == None
|
assert test1.segment_id_for_pos(3) == None
|
||||||
assert test1.segment_id_for_pos(10) == 's2'
|
assert test1.segment_id_for_pos(10) == 's2'
|
||||||
|
|
||||||
|
# ExtractedTextSegment('foo', unicodedata.normalize('NFD', 'Schlyñ'))
|
||||||
|
ExtractedTextSegment('foo', unicodedata.normalize('NFC', 'Schlyñ'))
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue