From ac8740c33fba027199699837c14b14a2f5639491 Mon Sep 17 00:00:00 2001 From: Mike Gerber Date: Thu, 12 Jun 2025 09:42:29 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=94=20=20Test=20if=20dtypes=20are=20as=20?= =?UTF-8?q?expected=20in=20produced=20Parquet=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- check_dtypes.py | 87 ----------------------- src/mods4pandas/alto4pandas.py | 5 +- src/mods4pandas/mods4pandas.py | 6 +- src/mods4pandas/tests/test_alto.py | 53 +++++++++++++- src/mods4pandas/tests/test_mods4pandas.py | 71 +++++++++++++++++- 5 files changed, 130 insertions(+), 92 deletions(-) delete mode 100644 check_dtypes.py diff --git a/check_dtypes.py b/check_dtypes.py deleted file mode 100644 index b5736df..0000000 --- a/check_dtypes.py +++ /dev/null @@ -1,87 +0,0 @@ -import re -import warnings -import os - -with warnings.catch_warnings(): - # Filter warnings on WSL - if "Microsoft" in os.uname().release: - warnings.simplefilter("ignore") - import pandas as pd - - -mods_info = pd.read_parquet("mods_info_df.parquet") -page_info = pd.read_parquet("page_info_df.parquet") -alto_info = pd.read_parquet("alto_info_df.parquet") - -# Check -EXPECTED_TYPES = { - - # mods_info - - r"mets_file": ("object", ["str"]), - r"titleInfo_title": ("object", ["str"]), - r"titleInfo_subTitle": ("object", ["str", "NoneType"]), - r"titleInfo_partName": ("object", ["str", "NoneType"]), - r"identifier-.*": ("object", ["str", "NoneType"]), - r"location_.*": ("object", ["str", "NoneType"]), - r"name\d+_.*roleTerm": ("object", ["ndarray", "NoneType"]), - r"name\d+_.*": ("object", ["str", "NoneType"]), - r"relatedItem-.*_recordInfo_recordIdentifier": ("object", ["str", "NoneType"]), - r"typeOfResource": ("object", ["str", "NoneType"]), - r"accessCondition-.*": ("object", ["str", "NoneType"]), - r"originInfo-.*": ("object", ["str", "NoneType"]), - - r".*-count": ("Int64", None), - - r"genre-.*": ("object", ["ndarray", "NoneType"]), - r"subject-.*": ("object", ["ndarray", "NoneType"]), - r"language_.*Term": ("object", ["ndarray", "NoneType"]), - r"classification-.*": ("object", ["ndarray", "NoneType"]), - - # page_info - - r"fileGrp_.*_file_FLocat_href": ("object", ["str", "NoneType"]), - r"structMap-LOGICAL_TYPE_.*": ("boolean", None), - - # alto_info - - r"Description_.*": ("object", ["str", "NoneType"]), - r"Layout_Page_ID": ("object", ["str", "NoneType"]), - r"Layout_Page_PHYSICAL_(IMG|IMAGE)_NR": ("object", ["str", "NoneType"]), - r"Layout_Page_PROCESSING": ("object", ["str", "NoneType"]), - r"Layout_Page_QUALITY": ("object", ["str", "NoneType"]), - r"Layout_Page_//alto:String/@WC-.*": ("Float64", None), - r"alto_xmlns": ("object", ["str", "NoneType"]), - - r"Layout_Page_(WIDTH|HEIGHT)": ("Int64", None), -} - -def expected_types(c): - for r, types in EXPECTED_TYPES.items(): - if re.fullmatch(r, c): - edt = types[0] - einner_types = types[1] - if einner_types: - einner_types = set(einner_types) - return edt, einner_types - return None, None - -def check_types(df): - for c in df.columns: - dt = df.dtypes[c] - edt, einner_types = expected_types(c) - - if edt is None: - print(f"No expected dtype known for column {c} (got {dt})") - elif dt != edt: - print(f"Unexpected dtype {dt} for column {c} (expected {edt})") - - if edt == "object": - inner_types = set(type(v).__name__ for v in df[c]) - if any(it not in einner_types for it in inner_types): - print(f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})") - -check_types(mods_info) -check_types(page_info) -check_types(alto_info) - diff --git a/src/mods4pandas/alto4pandas.py b/src/mods4pandas/alto4pandas.py index 1d7b748..359a26e 100755 --- a/src/mods4pandas/alto4pandas.py +++ b/src/mods4pandas/alto4pandas.py @@ -138,7 +138,7 @@ def walk(m): @click.argument('alto_files', type=click.Path(exists=True), required=True, nargs=-1) @click.option('--output', '-o', 'output_file', type=click.Path(), help='Output Parquet file', default='alto_info_df.parquet', show_default=True) -def process(alto_files: List[str], output_file: str): +def process_command(alto_files: List[str], output_file: str): """ A tool to convert the ALTO metadata in INPUT to a pandas DataFrame. @@ -151,6 +151,9 @@ def process(alto_files: List[str], output_file: str): - and a CSV file with all conversion warnings. """ + process(alto_files, output_file) + +def process(alto_files: List[str], output_file: str): # Extend file list if directories are given alto_files_real = [] for m in alto_files: diff --git a/src/mods4pandas/mods4pandas.py b/src/mods4pandas/mods4pandas.py index 7d45b47..669c1e0 100755 --- a/src/mods4pandas/mods4pandas.py +++ b/src/mods4pandas/mods4pandas.py @@ -382,7 +382,7 @@ def pages_to_dict(mets, raise_errors=True) -> List[Dict]: @click.option('--output', '-o', 'output_file', type=click.Path(), help='Output Parquet file', default='mods_info_df.parquet', show_default=True) @click.option('--output-page-info', type=click.Path(), help='Output page info Parquet file') -def process(mets_files: list[str], output_file: str, output_page_info: str): +def process_command(mets_files: list[str], output_file: str, output_page_info: str): """ A tool to convert the MODS metadata in INPUT to a pandas DataFrame. @@ -393,7 +393,9 @@ def process(mets_files: list[str], output_file: str, output_page_info: str): Per-page information (e.g. structure information) can be output to a separate Parquet file. """ + process(mets_files, output_file, output_page_info) +def process(mets_files: list[str], output_file: str, output_page_info: str): # Extend file list if directories are given mets_files_real: list[str] = [] for m in mets_files: @@ -476,7 +478,7 @@ def main(): for prefix, uri in ns.items(): ET.register_namespace(prefix, uri) - process() + process_command() if __name__ == '__main__': diff --git a/src/mods4pandas/tests/test_alto.py b/src/mods4pandas/tests/test_alto.py index 827bc7a..adf931f 100644 --- a/src/mods4pandas/tests/test_alto.py +++ b/src/mods4pandas/tests/test_alto.py @@ -1,9 +1,13 @@ +from pathlib import Path +import re from lxml import etree as ET +import pandas as pd -from mods4pandas.alto4pandas import alto_to_dict +from mods4pandas.alto4pandas import alto_to_dict, process from mods4pandas.lib import flatten +TESTS_DATA_DIR = Path(__file__).parent / "data" def dict_fromstring(x): return flatten(alto_to_dict(ET.fromstring(x))) @@ -79,3 +83,50 @@ def test_String_TAGREF_counts(): """) assert d['Layout_Page_//alto:String[@TAGREFS]-count'] == 3 assert d['Layout_Page_String-count'] == 4 + + +def test_dtypes(tmp_path): + alto_dir = (TESTS_DATA_DIR / "alto").absolute().as_posix() + alto_info_df_parquet = (tmp_path / "test_dtypes_alto_info.parquet").as_posix() + process([alto_dir], alto_info_df_parquet) + alto_info_df = pd.read_parquet(alto_info_df_parquet) + + EXPECTED_TYPES = { + r"Description_.*": ("object", ["str", "NoneType"]), + r"Layout_Page_ID": ("object", ["str", "NoneType"]), + r"Layout_Page_PHYSICAL_(IMG|IMAGE)_NR": ("object", ["str", "NoneType"]), + r"Layout_Page_PROCESSING": ("object", ["str", "NoneType"]), + r"Layout_Page_QUALITY": ("object", ["str", "NoneType"]), + r"Layout_Page_//alto:String/@WC-.*": ("Float64", None), + r".*-count": ("Int64", None), + r"alto_xmlns": ("object", ["str", "NoneType"]), + + r"Layout_Page_(WIDTH|HEIGHT)": ("Int64", None), + } + def expected_types(c): + """Return the expected types for column c.""" + for r, types in EXPECTED_TYPES.items(): + if re.fullmatch(r, c): + edt = types[0] + einner_types = types[1] + if einner_types: + einner_types = set(einner_types) + return edt, einner_types + return None, None + + def check_types(df): + """Check the types of the DataFrame df.""" + for c in df.columns: + dt = df.dtypes[c] + edt, einner_types = expected_types(c) + print(c, dt, edt) + + assert edt is not None, f"No expected dtype known for column {c} (got {dt})" + assert dt == edt, f"Unexpected dtype {dt} for column {c} (expected {edt})" + + if edt == "object": + inner_types = set(type(v).__name__ for v in df[c]) + assert all(it in einner_types for it in inner_types), \ + f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})" + + check_types(alto_info_df) \ No newline at end of file diff --git a/src/mods4pandas/tests/test_mods4pandas.py b/src/mods4pandas/tests/test_mods4pandas.py index f9a98d7..0707a74 100644 --- a/src/mods4pandas/tests/test_mods4pandas.py +++ b/src/mods4pandas/tests/test_mods4pandas.py @@ -1,10 +1,14 @@ +from pathlib import Path +import re from lxml import etree as ET +import pandas as pd import pytest -from mods4pandas.mods4pandas import mods_to_dict +from mods4pandas.mods4pandas import mods_to_dict, process from mods4pandas.lib import flatten +TESTS_DATA_DIR = Path(__file__).parent / "data" def dict_fromstring(x): """Helper function to parse a MODS XML string to a flattened dict""" @@ -151,3 +155,68 @@ def test_relatedItem(): """) assert d['relatedItem-original_recordInfo_recordIdentifier-dnb-ppn'] == '1236513355' + +def test_dtypes(tmp_path): + mets_files = [p.absolute().as_posix() for p in (TESTS_DATA_DIR / "mets-mods").glob("*.xml")] + mods_info_df_parquet = (tmp_path / "test_dtypes_mods_info.parquet").as_posix() + page_info_df_parquet = (tmp_path / "test_dtypes_page_info.parquet").as_posix() + process(mets_files, mods_info_df_parquet, page_info_df_parquet) + mods_info_df = pd.read_parquet(mods_info_df_parquet) + page_info_df = pd.read_parquet(page_info_df_parquet) + + EXPECTED_TYPES = { + # mods_info + + r"mets_file": ("object", ["str"]), + r"titleInfo_title": ("object", ["str"]), + r"titleInfo_subTitle": ("object", ["str", "NoneType"]), + r"titleInfo_partName": ("object", ["str", "NoneType"]), + r"identifier-.*": ("object", ["str", "NoneType"]), + r"location_.*": ("object", ["str", "NoneType"]), + r"name\d+_.*roleTerm": ("object", ["ndarray", "NoneType"]), + r"name\d+_.*": ("object", ["str", "NoneType"]), + r"relatedItem-.*_recordInfo_recordIdentifier": ("object", ["str", "NoneType"]), + r"typeOfResource": ("object", ["str", "NoneType"]), + r"accessCondition-.*": ("object", ["str", "NoneType"]), + r"originInfo-.*": ("object", ["str", "NoneType"]), + + r".*-count": ("Int64", None), + + r"genre-.*": ("object", ["ndarray", "NoneType"]), + r"subject-.*": ("object", ["ndarray", "NoneType"]), + r"language_.*Term": ("object", ["ndarray", "NoneType"]), + r"classification-.*": ("object", ["ndarray", "NoneType"]), + + # page_info + + r"fileGrp_.*_file_FLocat_href": ("object", ["str", "NoneType"]), + r"structMap-LOGICAL_TYPE_.*": ("boolean", None), + } + def expected_types(c): + """Return the expected types for column c.""" + for r, types in EXPECTED_TYPES.items(): + if re.fullmatch(r, c): + edt = types[0] + einner_types = types[1] + if einner_types: + einner_types = set(einner_types) + return edt, einner_types + return None, None + + def check_types(df): + """Check the types of the DataFrame df.""" + for c in df.columns: + dt = df.dtypes[c] + edt, einner_types = expected_types(c) + print(c, dt, edt) + + assert edt is not None, f"No expected dtype known for column {c} (got {dt})" + assert dt == edt, f"Unexpected dtype {dt} for column {c} (expected {edt})" + + if edt == "object": + inner_types = set(type(v).__name__ for v in df[c]) + assert all(it in einner_types for it in inner_types), \ + f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})" + + check_types(mods_info_df) + check_types(page_info_df) \ No newline at end of file