|
|
@ -13,12 +13,13 @@ def test_text():
|
|
|
|
test1 = ExtractedText(
|
|
|
|
test1 = ExtractedText(
|
|
|
|
None,
|
|
|
|
None,
|
|
|
|
[
|
|
|
|
[
|
|
|
|
ExtractedText("s0", None, None, "foo"),
|
|
|
|
ExtractedText("s0", None, None, "foo", grapheme_clusters("foo")),
|
|
|
|
ExtractedText("s1", None, None, "bar"),
|
|
|
|
ExtractedText("s1", None, None, "bar", grapheme_clusters("bar")),
|
|
|
|
ExtractedText("s2", None, None, "bazinga"),
|
|
|
|
ExtractedText("s2", None, None, "bazinga", grapheme_clusters("bazinga")),
|
|
|
|
],
|
|
|
|
],
|
|
|
|
" ",
|
|
|
|
" ",
|
|
|
|
None,
|
|
|
|
None,
|
|
|
|
|
|
|
|
None,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
assert test1.text == "foo bar bazinga"
|
|
|
|
assert test1.text == "foo bar bazinga"
|
|
|
@ -29,8 +30,12 @@ def test_text():
|
|
|
|
|
|
|
|
|
|
|
|
def test_normalization_check():
|
|
|
|
def test_normalization_check():
|
|
|
|
with pytest.raises(ValueError, match=r".*is not in NFC.*"):
|
|
|
|
with pytest.raises(ValueError, match=r".*is not in NFC.*"):
|
|
|
|
ExtractedText("foo", None, None, unicodedata.normalize("NFD", "Schlyñ"))
|
|
|
|
ExtractedText("foo", None, None,
|
|
|
|
assert ExtractedText("foo", None, None, unicodedata.normalize("NFC", "Schlyñ"))
|
|
|
|
unicodedata.normalize("NFD", "Schlyñ"),
|
|
|
|
|
|
|
|
grapheme_clusters(unicodedata.normalize("NFD", "Schlyñ")))
|
|
|
|
|
|
|
|
assert ExtractedText("foo", None, None,
|
|
|
|
|
|
|
|
unicodedata.normalize("NFC", "Schlyñ"),
|
|
|
|
|
|
|
|
grapheme_clusters(unicodedata.normalize("NFC", "Schlyñ")))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
AlignmentElement = namedtuple("AlignmentElement", "left right left_id right_id")
|
|
|
|
AlignmentElement = namedtuple("AlignmentElement", "left right left_id right_id")
|
|
|
@ -47,25 +52,27 @@ def test_align():
|
|
|
|
test1 = ExtractedText(
|
|
|
|
test1 = ExtractedText(
|
|
|
|
None,
|
|
|
|
None,
|
|
|
|
[
|
|
|
|
[
|
|
|
|
ExtractedText("s0", None, None, "foo"),
|
|
|
|
ExtractedText("s0", None, None, "foo", grapheme_clusters("foo")),
|
|
|
|
ExtractedText("s1", None, None, "bar"),
|
|
|
|
ExtractedText("s1", None, None, "bar", grapheme_clusters("bar")),
|
|
|
|
ExtractedText("s2", None, None, "batzinga"),
|
|
|
|
ExtractedText("s2", None, None, "batzinga", grapheme_clusters("batzinga")),
|
|
|
|
],
|
|
|
|
],
|
|
|
|
" ",
|
|
|
|
" ",
|
|
|
|
None,
|
|
|
|
None,
|
|
|
|
|
|
|
|
None,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
test2 = ExtractedText(
|
|
|
|
test2 = ExtractedText(
|
|
|
|
None,
|
|
|
|
None,
|
|
|
|
[
|
|
|
|
[
|
|
|
|
ExtractedText("x0", None, None, "foo"),
|
|
|
|
ExtractedText("x0", None, None, "foo", grapheme_clusters("foo")),
|
|
|
|
ExtractedText("x1", None, None, "bar"),
|
|
|
|
ExtractedText("x1", None, None, "bar", grapheme_clusters("bar")),
|
|
|
|
# extra .
|
|
|
|
# extra .
|
|
|
|
ExtractedText("x2", None, None, "."),
|
|
|
|
ExtractedText("x2", None, None, ".", grapheme_clusters(".")),
|
|
|
|
# deletion + different grapheme cluster, m̃ also is two Python characters
|
|
|
|
# deletion + different grapheme cluster, m̃ also is two Python characters
|
|
|
|
ExtractedText("x3", None, None, "bazim̃ga"),
|
|
|
|
ExtractedText("x3", None, None, "bazim̃ga", grapheme_clusters("bazim̃ga")),
|
|
|
|
],
|
|
|
|
],
|
|
|
|
" ",
|
|
|
|
" ",
|
|
|
|
None,
|
|
|
|
None,
|
|
|
|
|
|
|
|
None,
|
|
|
|
)
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
left_pos = 0
|
|
|
|
left_pos = 0
|
|
|
|