qurator-spk/dinglehopper
1
0
Fork 0
mirror of https://github.com/qurator-spk/dinglehopper.git synced 2025-06-07 19:05:13 +02:00
Code Issues Releases Wiki Activity

opt out of 7f8a8dd5 (uniseg update that requires py39)

Browse source
This commit is contained in:
Robert Sachunsky 2025-04-17 16:08:41 +02:00
parent f6dfb77f94
commit 64444dd419
2 changed files with 7 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
requirements.txt
View file

@ -1,7 +1,7 @@
click click
jinja2 jinja2
lxml lxml
uniseg >= 0.9.1 uniseg >= 0.8.0
numpy numpy
colorama colorama
MarkupSafe MarkupSafe

7
src/dinglehopper/word_error_rate.py
View file

@ -21,10 +21,15 @@ def patch_word_break():
https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt
""" """
old_word_break = uniseg.wordbreak.word_break old_word_break = uniseg.wordbreak.word_break
if hasattr(uniseg.wordbreak, 'Word_Break'):
aletter = uniseg.wordbreak.Word_Break.ALetter
else:
# uniseg<0.9
aletter = uniseg.wordbreak.WordBreak.ALETTER
def new_word_break(c): def new_word_break(c):
if 0xE000 <= ord(c) <= 0xF8FF: # Private Use Area if 0xE000 <= ord(c) <= 0xF8FF: # Private Use Area
return uniseg.wordbreak.Word_Break.ALetter return aletter
else: else:
return old_word_break(c) return old_word_break(c)