diff --git a/check_dtypes.py b/check_dtypes.py new file mode 100644 index 0000000..64fe514 --- /dev/null +++ b/check_dtypes.py @@ -0,0 +1,57 @@ +import pandas as pd +import re + + +# Fix +mods_info = pd.read_parquet("mods_info_df.parquet") +for c in mods_info.columns: + if c.endswith("-count"): + mods_info[c] = mods_info[c].astype('Int64') + + +# Tmp to parquet +mods_info.to_parquet("tmp.parquet") +mods_info = pd.read_parquet("tmp.parquet") + + +# Check +EXPECTED_TYPES = { + r"mets_file": ("object", ["str"]), + r"titleInfo_title": ("object", ["str"]), + r"titleInfo_subTitle": ("object", ["str", "NoneType"]), + r"titleInfo_partName": ("object", ["str", "NoneType"]), + r"identifier-.*": ("object", ["str", "NoneType"]), + r"location_.*t ": ("object", ["str", "NoneType"]), + r"name\d+_.*": ("object", ["str", "NoneType"]), + r"relatedItem-.*_recordInfo_recordIdentifier": ("object", ["str", "NoneType"]), + r".*-count": ("Int64", None), + + # XXX possibly sets: + r"genre-.*": ("object", ["str", "NoneType"]), + r"subject-.*": ("object", ["str", "NoneType"]), + r"language_.*Term": ("object", ["str", "NoneType"]), +} +def expected_types(c): + for r, types in EXPECTED_TYPES.items(): + if re.fullmatch(r, c): + edt = types[0] + einner_types = types[1] + if einner_types: + einner_types = set(einner_types) + return edt, einner_types + return None, None + +for c in mods_info.columns: + dt = mods_info.dtypes[c] + edt, einner_types = expected_types(c) + + if edt is None: + print(f"No expected dtype known for column {c}") + elif dt != edt: + print(f"Unexpected dtype {dt} for column {c} (expected {edt})") + + if edt == "object": + inner_types = set(type(v).__name__ for v in mods_info[c]) + if any(it not in einner_types for it in inner_types): + print(f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})") +