qurator-spk/dinglehopper
1
0
Fork 0
mirror of https://github.com/qurator-spk/dinglehopper.git synced 2025-06-09 11:50:00 +02:00
Code Issues Releases Wiki Activity

🐛 dinglehopper: Patch word_break only once
Some checks reported errors
continuous-integration/drone/push Build encountered an error
Details

Browse source 
Previously, we (accidently) patched uniseg's word_break on every call
to words(). Do it only once.
This commit is contained in:
Gerber, Mike 2022-01-24 18:44:30 +01:00
parent b6bde2b7ec
commit 8a3f5e48c2
1 changed files with 22 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

27
qurator/dinglehopper/word_error_rate.py
View file

@ -10,12 +10,17 @@ from rapidfuzz.string_metric import levenshtein
from . import ExtractedText
@multimethod
def words(s: str):
"""Extract words from a string"""
# Did we patch uniseg.wordbreak.word_break already?
word_break_patched = False
# Patch uniseg.wordbreak.word_break to deal with our private use characters. See also
# https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt
def patch_word_break():
"""
Patch uniseg.wordbreak.word_break to deal with our private use characters.
See also
https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt
"""
old_word_break = uniseg.wordbreak.word_break
def new_word_break(c, index=0):
@ -25,6 +30,18 @@ def words(s: str):
return old_word_break(c, index)
uniseg.wordbreak.word_break = new_word_break
global word_break_patched
word_break_patched = True
@multimethod
def words(s: str):
"""Extract words from a string"""
global word_break_patched
if not word_break_patched:
patch_word_break()
# Check if c is an unwanted character, i.e. whitespace, punctuation, or similar
def unwanted(c):