diff --git a/qurator/dinglehopper/__init__.py b/qurator/dinglehopper/__init__.py
index 8e58101..f883dfc 100644
--- a/qurator/dinglehopper/__init__.py
+++ b/qurator/dinglehopper/__init__.py
@@ -1,5 +1,3 @@
 from .ocr_files import *
 from .extracted_text import *
-from .character_error_rate import *
-from .word_error_rate import *
 from .align import *
diff --git a/qurator/dinglehopper/cli.py b/qurator/dinglehopper/cli.py
index 32e159f..c8cb2ef 100644
--- a/qurator/dinglehopper/cli.py
+++ b/qurator/dinglehopper/cli.py
@@ -6,8 +6,8 @@ from markupsafe import escape
 from uniseg.graphemecluster import grapheme_clusters
 from ocrd_utils import initLogging
 
-from .character_error_rate import character_error_rate_n
-from .word_error_rate import word_error_rate_n, words_normalized
+from .metrics.character_error_rate import character_error_rate_n
+from .metrics.word_error_rate import word_error_rate_n, words_normalized
 from .align import seq_align
 from .extracted_text import ExtractedText
 from .ocr_files import extract
diff --git a/qurator/dinglehopper/metrics/__init__.py b/qurator/dinglehopper/metrics/__init__.py
new file mode 100644
index 0000000..9f370c4
--- /dev/null
+++ b/qurator/dinglehopper/metrics/__init__.py
@@ -0,0 +1,2 @@
+from .character_error_rate import *
+from .word_error_rate import *
diff --git a/qurator/dinglehopper/character_error_rate.py b/qurator/dinglehopper/metrics/character_error_rate.py
similarity index 92%
rename from qurator/dinglehopper/character_error_rate.py
rename to qurator/dinglehopper/metrics/character_error_rate.py
index 0c3ef7d..4dae8ee 100644
--- a/qurator/dinglehopper/character_error_rate.py
+++ b/qurator/dinglehopper/metrics/character_error_rate.py
@@ -6,8 +6,8 @@ from typing import Tuple
 from multimethod import multimethod
 from uniseg.graphemecluster import grapheme_clusters
 
-from .edit_distance import distance
-from .extracted_text import ExtractedText
+from ..edit_distance import distance
+from ..extracted_text import ExtractedText
 
 
 @multimethod
diff --git a/qurator/dinglehopper/word_error_rate.py b/qurator/dinglehopper/metrics/word_error_rate.py
similarity index 97%
rename from qurator/dinglehopper/word_error_rate.py
rename to qurator/dinglehopper/metrics/word_error_rate.py
index dde57b9..5a42eee 100644
--- a/qurator/dinglehopper/word_error_rate.py
+++ b/qurator/dinglehopper/metrics/word_error_rate.py
@@ -6,8 +6,8 @@ from multimethod import multimethod
 
 import uniseg.wordbreak
 
-from .edit_distance import levenshtein
-from . import ExtractedText
+from ..edit_distance import levenshtein
+from .. import ExtractedText
 
 
 @multimethod
diff --git a/qurator/dinglehopper/tests/metrics/__init__.py b/qurator/dinglehopper/tests/metrics/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/qurator/dinglehopper/tests/test_character_error_rate.py b/qurator/dinglehopper/tests/metrics/test_character_error_rate.py
similarity index 96%
rename from qurator/dinglehopper/tests/test_character_error_rate.py
rename to qurator/dinglehopper/tests/metrics/test_character_error_rate.py
index 39301b4..15d2e06 100644
--- a/qurator/dinglehopper/tests/test_character_error_rate.py
+++ b/qurator/dinglehopper/tests/metrics/test_character_error_rate.py
@@ -3,7 +3,7 @@ from __future__ import division, print_function
 import math
 import unicodedata
 
-from .. import character_error_rate
+from ...metrics import character_error_rate
 
 
 def test_character_error_rate():
diff --git a/qurator/dinglehopper/tests/test_integ_character_error_rate_ocr.py b/qurator/dinglehopper/tests/metrics/test_integ_character_error_rate_ocr.py
similarity index 94%
rename from qurator/dinglehopper/tests/test_integ_character_error_rate_ocr.py
rename to qurator/dinglehopper/tests/metrics/test_integ_character_error_rate_ocr.py
index e307a84..1ea07fe 100644
--- a/qurator/dinglehopper/tests/test_integ_character_error_rate_ocr.py
+++ b/qurator/dinglehopper/tests/metrics/test_integ_character_error_rate_ocr.py
@@ -6,9 +6,10 @@ import pytest
 from lxml import etree as ET
 from uniseg.graphemecluster import grapheme_clusters
 
-from .. import character_error_rate, page_text, alto_text
+from ... import page_text, alto_text
+from ...metrics import character_error_rate
 
-data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
+data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../", "data")
 
 
 @pytest.mark.integration
diff --git a/qurator/dinglehopper/tests/test_integ_word_error_rate_ocr.py b/qurator/dinglehopper/tests/metrics/test_integ_word_error_rate_ocr.py
similarity index 94%
rename from qurator/dinglehopper/tests/test_integ_word_error_rate_ocr.py
rename to qurator/dinglehopper/tests/metrics/test_integ_word_error_rate_ocr.py
index ba865b4..1b8dd7e 100644
--- a/qurator/dinglehopper/tests/test_integ_word_error_rate_ocr.py
+++ b/qurator/dinglehopper/tests/metrics/test_integ_word_error_rate_ocr.py
@@ -5,9 +5,10 @@ import os
 import pytest
 from lxml import etree as ET
 
-from .. import word_error_rate, words, page_text, alto_text
+from ... import page_text, alto_text
+from ...metrics import word_error_rate, words\
 
-data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
+data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../", "data")
 
 
 @pytest.mark.integration
diff --git a/qurator/dinglehopper/tests/test_word_error_rate.py b/qurator/dinglehopper/tests/metrics/test_word_error_rate.py
similarity index 97%
rename from qurator/dinglehopper/tests/test_word_error_rate.py
rename to qurator/dinglehopper/tests/metrics/test_word_error_rate.py
index bc7b91e..36f2823 100644
--- a/qurator/dinglehopper/tests/test_word_error_rate.py
+++ b/qurator/dinglehopper/tests/metrics/test_word_error_rate.py
@@ -2,7 +2,7 @@ from __future__ import division, print_function
 
 import math
 
-from .. import word_error_rate, words
+from ...metrics import word_error_rate, words
 
 
 def test_words():