mirror of
https://github.com/qurator-spk/dinglehopper.git
synced 2025-06-09 11:50:00 +02:00
🐛 dinglehopper: Patch word_break only once
Some checks reported errors
continuous-integration/drone/push Build encountered an error
Some checks reported errors
continuous-integration/drone/push Build encountered an error
Previously, we (accidently) patched uniseg's word_break on every call to words(). Do it only once.
This commit is contained in:
parent
b6bde2b7ec
commit
8a3f5e48c2
1 changed files with 22 additions and 5 deletions
|
@ -10,12 +10,17 @@ from rapidfuzz.string_metric import levenshtein
|
|||
from . import ExtractedText
|
||||
|
||||
|
||||
@multimethod
|
||||
def words(s: str):
|
||||
"""Extract words from a string"""
|
||||
# Did we patch uniseg.wordbreak.word_break already?
|
||||
word_break_patched = False
|
||||
|
||||
# Patch uniseg.wordbreak.word_break to deal with our private use characters. See also
|
||||
# https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt
|
||||
|
||||
def patch_word_break():
|
||||
"""
|
||||
Patch uniseg.wordbreak.word_break to deal with our private use characters.
|
||||
|
||||
See also
|
||||
https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/WordBreakProperty.txt
|
||||
"""
|
||||
old_word_break = uniseg.wordbreak.word_break
|
||||
|
||||
def new_word_break(c, index=0):
|
||||
|
@ -25,6 +30,18 @@ def words(s: str):
|
|||
return old_word_break(c, index)
|
||||
|
||||
uniseg.wordbreak.word_break = new_word_break
|
||||
global word_break_patched
|
||||
word_break_patched = True
|
||||
|
||||
|
||||
@multimethod
|
||||
def words(s: str):
|
||||
"""Extract words from a string"""
|
||||
|
||||
global word_break_patched
|
||||
if not word_break_patched:
|
||||
patch_word_break()
|
||||
|
||||
|
||||
# Check if c is an unwanted character, i.e. whitespace, punctuation, or similar
|
||||
def unwanted(c):
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue