🚧 Check dtypes (WIP)

check_dtypes.py

@@ -0,0 +1,57 @@
+import pandas as pd
+import re
+
+
+# Fix
+mods_info = pd.read_parquet("mods_info_df.parquet")
+for c in mods_info.columns:
+    if c.endswith("-count"):
+        mods_info[c] = mods_info[c].astype('Int64')
+
+
+# Tmp to parquet
+mods_info.to_parquet("tmp.parquet")
+mods_info = pd.read_parquet("tmp.parquet")
+
+
+# Check
+EXPECTED_TYPES = {
+        r"mets_file": ("object", ["str"]),
+        r"titleInfo_title": ("object", ["str"]),
+        r"titleInfo_subTitle": ("object", ["str", "NoneType"]),
+        r"titleInfo_partName": ("object", ["str", "NoneType"]),
+        r"identifier-.*": ("object", ["str", "NoneType"]),
+        r"location_.*t ": ("object", ["str", "NoneType"]),
+        r"name\d+_.*": ("object", ["str", "NoneType"]),
+        r"relatedItem-.*_recordInfo_recordIdentifier": ("object", ["str", "NoneType"]),
+        r".*-count": ("Int64", None),
+
+        # XXX possibly sets:
+        r"genre-.*": ("object", ["str", "NoneType"]),
+        r"subject-.*": ("object", ["str", "NoneType"]),
+        r"language_.*Term": ("object", ["str", "NoneType"]),
+}
+def expected_types(c):
+    for r, types in EXPECTED_TYPES.items():
+        if re.fullmatch(r, c):
+            edt = types[0]
+            einner_types = types[1]
+            if einner_types:
+                einner_types = set(einner_types)
+            return edt, einner_types
+    return None, None
+
+for c in mods_info.columns:
+    dt = mods_info.dtypes[c]
+    edt, einner_types = expected_types(c)
+
+    if edt is None:
+        print(f"No expected dtype known for column {c}")
+    elif dt != edt:
+        print(f"Unexpected dtype {dt} for column {c} (expected {edt})")
+
+    if edt == "object":
+        inner_types = set(type(v).__name__ for v in mods_info[c])
+        if any(it not in einner_types for it in inner_types):
+            print(f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})")
+