qurator-spk/modstool
1
0
Fork 0
mirror of https://github.com/qurator-spk/modstool.git synced 2025-06-25 19:49:54 +02:00
Code Issues Releases Wiki Activity

🚧 check_dtypes: Check alto_info types

Browse source
This commit is contained in:
Mike Gerber 2025-06-11 16:18:33 +02:00
parent e4db150cba
commit 5384e18ab5
1 changed files with 32 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

43
check_dtypes.py
View file

@ -3,9 +3,13 @@ import re
mods_info = pd.read_parquet("mods_info_df.parquet")
alto_info = pd.read_parquet("alto_info_df.parquet")
# Check
EXPECTED_TYPES = {
# mods_info
r"mets_file": ("object", ["str"]),
r"titleInfo_title": ("object", ["str"]),
r"titleInfo_subTitle": ("object", ["str", "NoneType"]),
@ -25,6 +29,19 @@ EXPECTED_TYPES = {
r"subject-.*": ("object", ["str", "NoneType"]),
r"language_.*Term": ("object", ["str", "NoneType"]),
r"classification-.*": ("object", ["str", "NoneType"]),
# alto_info
r"Description_.*": ("object", ["str", "NoneType"]),
r"Layout_Page_ID": ("object", ["str", "NoneType"]),
r"Layout_Page_PHYSICAL_(IMG|IMAGE)_NR": ("object", ["str", "NoneType"]),
r"Layout_Page_PROCESSING": ("object", ["str", "NoneType"]),
r"Layout_Page_QUALITY": ("object", ["str", "NoneType"]),
r"Layout_Page_//alto:String/@WC-.*": ("Float64", None),
r"alto_xmlns": ("object", ["str", "NoneType"]),
# XXX r"Layout_Page_(WIDTH|HEIGHT)": ("Int64", None),
r"Layout_Page_(WIDTH|HEIGHT)": ("object", ["str", "NoneType"]),
}
def expected_types(c):
for r, types in EXPECTED_TYPES.items():
@ -36,17 +53,21 @@ def expected_types(c):
return edt, einner_types
return None, None
for c in mods_info.columns:
dt = mods_info.dtypes[c]
edt, einner_types = expected_types(c)
def check_types(df):
for c in df.columns:
dt = df.dtypes[c]
edt, einner_types = expected_types(c)
if edt is None:
print(f"No expected dtype known for column {c}")
elif dt != edt:
print(f"Unexpected dtype {dt} for column {c} (expected {edt})")
if edt is None:
print(f"No expected dtype known for column {c}")
elif dt != edt:
print(f"Unexpected dtype {dt} for column {c} (expected {edt})")
if edt == "object":
inner_types = set(type(v).__name__ for v in mods_info[c])
if any(it not in einner_types for it in inner_types):
print(f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})")
if edt == "object":
inner_types = set(type(v).__name__ for v in df[c])
if any(it not in einner_types for it in inner_types):
print(f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})")
check_types(mods_info)
check_types(alto_info)