Implemented new metrics behaviour
Benjamin Rosemann
4 years ago
parent
9f5112f8f6
commit
06468a436e
@ -1,5 +1,5 @@
|
||||
from .bag_of_chars_accuracy import *
|
||||
from .bag_of_words_accuracy import *
|
||||
from .character_error_rate import *
|
||||
from .utils import Weights
|
||||
from .word_error_rate import *
|
||||
from .character_accuracy import *
|
||||
from .utils import MetricResult, Weights
|
||||
from .word_accuracy import *
|
||||
|
||||
@ -0,0 +1,28 @@
|
||||
from __future__ import division
|
||||
|
||||
from .utils import MetricResult, Weights
|
||||
from .. import distance
|
||||
from ..normalize import chars_normalized
|
||||
|
||||
|
||||
def character_accuracy(
|
||||
reference: str, compared: str, weights: Weights = Weights(1, 1, 1)
|
||||
) -> MetricResult:
|
||||
"""Compute character accuracy and error rate.
|
||||
|
||||
:return: NamedTuple representing the results of this metric.
|
||||
"""
|
||||
|
||||
weighted_errors = distance(reference, compared)
|
||||
n_ref = len(chars_normalized(reference))
|
||||
n_cmp = len(chars_normalized(compared))
|
||||
|
||||
return MetricResult(
|
||||
metric=character_accuracy.__name__,
|
||||
weights=weights,
|
||||
weighted_errors=int(weighted_errors),
|
||||
reference_elements=n_ref,
|
||||
compared_elements=n_cmp,
|
||||
)
|
||||
|
||||
# XXX Should we really count newlines here?
|
||||
@ -1,46 +0,0 @@
|
||||
from __future__ import division
|
||||
|
||||
from typing import Tuple
|
||||
|
||||
from multimethod import multimethod
|
||||
|
||||
from .. import distance
|
||||
from ..extracted_text import ExtractedText
|
||||
from ..normalize import chars_normalized
|
||||
|
||||
|
||||
@multimethod
|
||||
def character_error_rate_n(reference: str, compared: str) -> Tuple[float, int]:
|
||||
"""
|
||||
Compute character error rate.
|
||||
|
||||
:return: character error rate and length of the reference
|
||||
"""
|
||||
|
||||
d = distance(reference, compared)
|
||||
n = len(chars_normalized(reference))
|
||||
|
||||
if d == 0:
|
||||
return 0, n
|
||||
if n == 0:
|
||||
return float("inf"), n
|
||||
return d / n, n
|
||||
|
||||
# XXX Should we really count newlines here?
|
||||
|
||||
|
||||
@multimethod
|
||||
def character_error_rate_n(
|
||||
reference: ExtractedText, compared: ExtractedText
|
||||
) -> Tuple[float, int]:
|
||||
return character_error_rate_n(reference.text, compared.text)
|
||||
|
||||
|
||||
def character_error_rate(reference, compared) -> float:
|
||||
"""
|
||||
Compute character error rate.
|
||||
|
||||
:return: character error rate
|
||||
"""
|
||||
cer, _ = character_error_rate_n(reference, compared)
|
||||
return cer
|
||||
@ -0,0 +1,22 @@
|
||||
from .utils import MetricResult, Weights
|
||||
from ..edit_distance import levenshtein
|
||||
from ..normalize import words_normalized
|
||||
|
||||
|
||||
def word_accuracy(
|
||||
reference: str, compared: str, weights: Weights = Weights(1, 1, 1)
|
||||
) -> MetricResult:
|
||||
reference_seq = list(words_normalized(reference))
|
||||
compared_seq = list(words_normalized(compared))
|
||||
|
||||
weighted_errors = levenshtein(reference_seq, compared_seq)
|
||||
n_ref = len(reference_seq)
|
||||
n_cmp = len(compared_seq)
|
||||
|
||||
return MetricResult(
|
||||
metric=word_accuracy.__name__,
|
||||
weights=weights,
|
||||
weighted_errors=int(weighted_errors),
|
||||
reference_elements=n_ref,
|
||||
compared_elements=n_cmp,
|
||||
)
|
||||
@ -1,43 +0,0 @@
|
||||
from __future__ import division
|
||||
|
||||
from typing import Iterable, Tuple
|
||||
|
||||
from multimethod import multimethod
|
||||
|
||||
from ..edit_distance import levenshtein
|
||||
from ..extracted_text import ExtractedText
|
||||
from ..normalize import words_normalized
|
||||
|
||||
|
||||
@multimethod
|
||||
def word_error_rate_n(reference: str, compared: str) -> Tuple[float, int]:
|
||||
reference_seq = list(words_normalized(reference))
|
||||
compared_seq = list(words_normalized(compared))
|
||||
return word_error_rate_n(reference_seq, compared_seq)
|
||||
|
||||
|
||||
@multimethod
|
||||
def word_error_rate_n(
|
||||
reference: ExtractedText, compared: ExtractedText
|
||||
) -> Tuple[float, int]:
|
||||
return word_error_rate_n(reference.text, compared.text)
|
||||
|
||||
|
||||
@multimethod
|
||||
def word_error_rate_n(reference: Iterable, compared: Iterable) -> Tuple[float, int]:
|
||||
reference_seq = list(reference)
|
||||
compared_seq = list(compared)
|
||||
|
||||
d = levenshtein(reference_seq, compared_seq)
|
||||
n = len(reference_seq)
|
||||
|
||||
if d == 0:
|
||||
return 0, n
|
||||
if n == 0:
|
||||
return float("inf"), n
|
||||
return d / n, n
|
||||
|
||||
|
||||
def word_error_rate(reference, compared) -> float:
|
||||
wer, _ = word_error_rate_n(reference, compared)
|
||||
return wer
|
||||
@ -0,0 +1,40 @@
|
||||
import math
|
||||
import unicodedata
|
||||
|
||||
from ...metrics import character_accuracy
|
||||
|
||||
|
||||
def test_character_accuracy():
|
||||
assert character_accuracy("a", "a").error_rate == 0
|
||||
assert character_accuracy("a", "b").error_rate == 1 / 1
|
||||
assert character_accuracy("Foo", "Bar").error_rate == 3 / 3
|
||||
|
||||
assert character_accuracy("Foo", "").error_rate == 3 / 3
|
||||
|
||||
assert character_accuracy("", "").error_rate == 0
|
||||
assert math.isinf(character_accuracy("", "Foo").error_rate)
|
||||
|
||||
assert character_accuracy("Foo", "Food").error_rate == 1 / 3
|
||||
assert character_accuracy("Fnord", "Food").error_rate == 2 / 5
|
||||
assert character_accuracy("Müll", "Mull").error_rate == 1 / 4
|
||||
assert character_accuracy("Abstand", "Sand").error_rate == 4 / 7
|
||||
|
||||
|
||||
def test_character_accuracy_hard():
|
||||
s1 = unicodedata.normalize("NFC", "Schlyñ lorem ipsum.")
|
||||
s2 = unicodedata.normalize("NFD", "Schlyñ lorem ipsum!") # Different, decomposed!
|
||||
assert character_accuracy(s1, s2).error_rate == 1 / 19
|
||||
|
||||
s1 = "Schlyñ"
|
||||
assert (
|
||||
len(s1) == 6
|
||||
) # This ends with LATIN SMALL LETTER N WITH TILDE, so 6 code points
|
||||
s2 = "Schlym̃"
|
||||
assert (
|
||||
len(s2) == 7
|
||||
) # This, OTOH, ends with LATIN SMALL LETTER M + COMBINING TILDE, 7 code points
|
||||
|
||||
# Both strings have the same length in terms of grapheme clusters.
|
||||
# So the CER should be symmetrical.
|
||||
assert character_accuracy(s2, s1).error_rate == 1 / 6
|
||||
assert character_accuracy(s1, s2).error_rate == 1 / 6
|
||||
@ -1,41 +0,0 @@
|
||||
from __future__ import division, print_function
|
||||
|
||||
import math
|
||||
import unicodedata
|
||||
|
||||
from ...metrics import character_error_rate
|
||||
|
||||
|
||||
def test_character_error_rate():
|
||||
assert character_error_rate("a", "a") == 0
|
||||
assert character_error_rate("a", "b") == 1 / 1
|
||||
assert character_error_rate("Foo", "Bar") == 3 / 3
|
||||
|
||||
assert character_error_rate("Foo", "") == 3 / 3
|
||||
|
||||
assert character_error_rate("", "") == 0
|
||||
assert math.isinf(character_error_rate("", "Foo"))
|
||||
|
||||
assert character_error_rate("Foo", "Food") == 1 / 3
|
||||
assert character_error_rate("Fnord", "Food") == 2 / 5
|
||||
assert character_error_rate("Müll", "Mull") == 1 / 4
|
||||
assert character_error_rate("Abstand", "Sand") == 4 / 7
|
||||
|
||||
|
||||
def test_character_error_rate_hard():
|
||||
s1 = unicodedata.normalize("NFC", "Schlyñ lorem ipsum.")
|
||||
s2 = unicodedata.normalize("NFD", "Schlyñ lorem ipsum!") # Different, decomposed!
|
||||
assert character_error_rate(s1, s2) == 1 / 19
|
||||
|
||||
s1 = "Schlyñ"
|
||||
assert (
|
||||
len(s1) == 6
|
||||
) # This ends with LATIN SMALL LETTER N WITH TILDE, so 6 code points
|
||||
s2 = "Schlym̃"
|
||||
assert (
|
||||
len(s2) == 7
|
||||
) # This, OTOH, ends with LATIN SMALL LETTER M + COMBINING TILDE, 7 code points
|
||||
|
||||
# Both strings have the same length in terms of grapheme clusters. So the CER should be symmetrical.
|
||||
assert character_error_rate(s2, s1) == 1 / 6
|
||||
assert character_error_rate(s1, s2) == 1 / 6
|
||||
Loading…
Reference in New Issue