🚧 Check dtypes (WIP)
parent
88a6c5f26f
commit
be1c8609a3
@ -0,0 +1,57 @@
|
||||
import pandas as pd
|
||||
import re
|
||||
|
||||
|
||||
# Fix
|
||||
mods_info = pd.read_parquet("mods_info_df.parquet")
|
||||
for c in mods_info.columns:
|
||||
if c.endswith("-count"):
|
||||
mods_info[c] = mods_info[c].astype('Int64')
|
||||
|
||||
|
||||
# Tmp to parquet
|
||||
mods_info.to_parquet("tmp.parquet")
|
||||
mods_info = pd.read_parquet("tmp.parquet")
|
||||
|
||||
|
||||
# Check
|
||||
EXPECTED_TYPES = {
|
||||
r"mets_file": ("object", ["str"]),
|
||||
r"titleInfo_title": ("object", ["str"]),
|
||||
r"titleInfo_subTitle": ("object", ["str", "NoneType"]),
|
||||
r"titleInfo_partName": ("object", ["str", "NoneType"]),
|
||||
r"identifier-.*": ("object", ["str", "NoneType"]),
|
||||
r"location_.*t ": ("object", ["str", "NoneType"]),
|
||||
r"name\d+_.*": ("object", ["str", "NoneType"]),
|
||||
r"relatedItem-.*_recordInfo_recordIdentifier": ("object", ["str", "NoneType"]),
|
||||
r".*-count": ("Int64", None),
|
||||
|
||||
# XXX possibly sets:
|
||||
r"genre-.*": ("object", ["str", "NoneType"]),
|
||||
r"subject-.*": ("object", ["str", "NoneType"]),
|
||||
r"language_.*Term": ("object", ["str", "NoneType"]),
|
||||
}
|
||||
def expected_types(c):
|
||||
for r, types in EXPECTED_TYPES.items():
|
||||
if re.fullmatch(r, c):
|
||||
edt = types[0]
|
||||
einner_types = types[1]
|
||||
if einner_types:
|
||||
einner_types = set(einner_types)
|
||||
return edt, einner_types
|
||||
return None, None
|
||||
|
||||
for c in mods_info.columns:
|
||||
dt = mods_info.dtypes[c]
|
||||
edt, einner_types = expected_types(c)
|
||||
|
||||
if edt is None:
|
||||
print(f"No expected dtype known for column {c}")
|
||||
elif dt != edt:
|
||||
print(f"Unexpected dtype {dt} for column {c} (expected {edt})")
|
||||
|
||||
if edt == "object":
|
||||
inner_types = set(type(v).__name__ for v in mods_info[c])
|
||||
if any(it not in einner_types for it in inner_types):
|
||||
print(f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})")
|
||||
|
Loading…
Reference in New Issue