mirror of
https://github.com/qurator-spk/modstool.git
synced 2025-06-26 12:09:55 +02:00
🎨 Reformat (Black)
This commit is contained in:
parent
5c9858a061
commit
212df99436
7 changed files with 639 additions and 355 deletions
|
|
@ -9,14 +9,17 @@ from mods4pandas.lib import flatten
|
|||
|
||||
TESTS_DATA_DIR = Path(__file__).parent / "data"
|
||||
|
||||
|
||||
def dict_fromstring(x):
|
||||
return flatten(alto_to_dict(ET.fromstring(x)))
|
||||
return flatten(alto_to_dict(ET.fromstring(x)))
|
||||
|
||||
|
||||
def test_Page_counts():
|
||||
"""
|
||||
Elements below Layout/Page should be counted
|
||||
"""
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<alto xmlns="http://www.loc.gov/standards/alto/ns-v2#">
|
||||
<Layout>
|
||||
<Page ID="Page1" PHYSICAL_IMG_NR="1">
|
||||
|
|
@ -37,13 +40,16 @@ def test_Page_counts():
|
|||
</Page>
|
||||
</Layout>
|
||||
</alto>
|
||||
""")
|
||||
assert d['Layout_Page_TextBlock-count'] == 1
|
||||
assert d['Layout_Page_TextLine-count'] == 3
|
||||
assert d['Layout_Page_String-count'] == 6
|
||||
"""
|
||||
)
|
||||
assert d["Layout_Page_TextBlock-count"] == 1
|
||||
assert d["Layout_Page_TextLine-count"] == 3
|
||||
assert d["Layout_Page_String-count"] == 6
|
||||
|
||||
|
||||
def test_Tags_counts():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<alto xmlns="http://www.loc.gov/standards/alto/ns-v2#">
|
||||
<Tags>
|
||||
<NamedEntityTag ID="PER0" LABEL="Pentlings"/>
|
||||
|
|
@ -57,11 +63,14 @@ def test_Tags_counts():
|
|||
<NamedEntityTag ID="PER10" LABEL="Jhesu Christi"/>
|
||||
</Tags>
|
||||
</alto>
|
||||
""")
|
||||
assert d['Tags_NamedEntityTag-count'] == 9
|
||||
"""
|
||||
)
|
||||
assert d["Tags_NamedEntityTag-count"] == 9
|
||||
|
||||
|
||||
def test_String_TAGREF_counts():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<alto xmlns="http://www.loc.gov/standards/alto/ns-v2#">
|
||||
<Layout>
|
||||
<Page>
|
||||
|
|
@ -80,9 +89,10 @@ def test_String_TAGREF_counts():
|
|||
</Page>
|
||||
</Layout>
|
||||
</alto>
|
||||
""")
|
||||
assert d['Layout_Page_//alto:String[@TAGREFS]-count'] == 3
|
||||
assert d['Layout_Page_String-count'] == 4
|
||||
"""
|
||||
)
|
||||
assert d["Layout_Page_//alto:String[@TAGREFS]-count"] == 3
|
||||
assert d["Layout_Page_String-count"] == 4
|
||||
|
||||
|
||||
def test_dtypes(tmp_path):
|
||||
|
|
@ -100,9 +110,9 @@ def test_dtypes(tmp_path):
|
|||
r"Layout_Page_//alto:String/@WC-.*": ("Float64", None),
|
||||
r".*-count": ("Int64", None),
|
||||
r"alto_xmlns": ("object", ["str", "NoneType"]),
|
||||
|
||||
r"Layout_Page_(WIDTH|HEIGHT)": ("Int64", None),
|
||||
}
|
||||
|
||||
def expected_types(c):
|
||||
"""Return the expected types for column c."""
|
||||
for r, types in EXPECTED_TYPES.items():
|
||||
|
|
@ -126,7 +136,8 @@ def test_dtypes(tmp_path):
|
|||
|
||||
if edt == "object":
|
||||
inner_types = set(type(v).__name__ for v in df[c])
|
||||
assert all(it in einner_types for it in inner_types), \
|
||||
f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})"
|
||||
assert all(
|
||||
it in einner_types for it in inner_types
|
||||
), f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})"
|
||||
|
||||
check_types(alto_info_df)
|
||||
check_types(alto_info_df)
|
||||
|
|
|
|||
|
|
@ -6,15 +6,17 @@ from mods4pandas.lib import flatten
|
|||
|
||||
|
||||
def dict_fromstring(x):
|
||||
"""Helper function to parse a METS/MODS XML string to a flattened dict"""
|
||||
return flatten(mets_to_dict(ET.fromstring(x)))
|
||||
# XXX move to test lib
|
||||
"""Helper function to parse a METS/MODS XML string to a flattened dict"""
|
||||
return flatten(mets_to_dict(ET.fromstring(x)))
|
||||
# XXX move to test lib
|
||||
|
||||
|
||||
def test_fileGrp():
|
||||
"""
|
||||
Elements of mets:fileGrp should be counted
|
||||
"""
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mets:mets xmlns:mets="http://www.loc.gov/METS/">
|
||||
|
||||
<mets:fileSec>
|
||||
|
|
@ -31,5 +33,6 @@ def test_fileGrp():
|
|||
</mets:fileGrp>
|
||||
</mets:fileSec>
|
||||
</mets:mets>
|
||||
""")
|
||||
assert d['fileSec_fileGrp-PRESENTATION-count'] == 3
|
||||
"""
|
||||
)
|
||||
assert d["fileSec_fileGrp-PRESENTATION-count"] == 3
|
||||
|
|
|
|||
|
|
@ -10,36 +10,45 @@ from mods4pandas.lib import flatten
|
|||
|
||||
TESTS_DATA_DIR = Path(__file__).parent / "data"
|
||||
|
||||
|
||||
def dict_fromstring(x):
|
||||
"""Helper function to parse a MODS XML string to a flattened dict"""
|
||||
return flatten(mods_to_dict(ET.fromstring(x)))
|
||||
|
||||
|
||||
def test_single_language_languageTerm():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:language>
|
||||
<mods:languageTerm authority="iso639-2b" type="code">lat</mods:languageTerm>
|
||||
<mods:languageTerm authority="iso639-2b" type="code">ger</mods:languageTerm>
|
||||
</mods:language>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['language_languageTerm'] == {'ger', 'lat'}
|
||||
"""
|
||||
)
|
||||
assert d["language_languageTerm"] == {"ger", "lat"}
|
||||
|
||||
|
||||
def test_multitple_language_languageTerm():
|
||||
"""
|
||||
Different languages MAY have multiple mods:language elements.
|
||||
See MODS-AP 2.3.1
|
||||
"""
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:language><mods:languageTerm authority="iso639-2b" type="code">lat</mods:languageTerm></mods:language>
|
||||
<mods:language><mods:languageTerm authority="iso639-2b" type="code">ger</mods:languageTerm></mods:language>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['language_languageTerm'] == {'ger', 'lat'}
|
||||
"""
|
||||
)
|
||||
assert d["language_languageTerm"] == {"ger", "lat"}
|
||||
|
||||
|
||||
def test_role_roleTerm():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:name type="personal" valueURI="http://d-nb.info/gnd/117357669">
|
||||
<mods:displayForm>Wurm, Mary</mods:displayForm>
|
||||
|
|
@ -51,14 +60,17 @@ def test_role_roleTerm():
|
|||
</mods:role>
|
||||
</mods:name>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['name0_role_roleTerm'] == {'cmp'}
|
||||
"""
|
||||
)
|
||||
assert d["name0_role_roleTerm"] == {"cmp"}
|
||||
|
||||
|
||||
def test_multiple_role_roleTerm():
|
||||
"""
|
||||
Multiple mods:role/mods:roleTerm should be merged into one column.
|
||||
"""
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:name type="personal" valueURI="http://d-nb.info/gnd/117357669">
|
||||
<mods:displayForm>Wurm, Mary</mods:displayForm>
|
||||
|
|
@ -73,8 +85,10 @@ def test_multiple_role_roleTerm():
|
|||
</mods:role>
|
||||
</mods:name>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['name0_role_roleTerm'] == {'cmp', 'aut'}
|
||||
"""
|
||||
)
|
||||
assert d["name0_role_roleTerm"] == {"cmp", "aut"}
|
||||
|
||||
|
||||
def test_scriptTerm():
|
||||
"""
|
||||
|
|
@ -82,7 +96,8 @@ def test_scriptTerm():
|
|||
|
||||
See MODS-AP 2.3.1.
|
||||
"""
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:language>
|
||||
<mods:languageTerm authority="iso639-2b" type="code">ger</mods:languageTerm>
|
||||
|
|
@ -94,44 +109,59 @@ def test_scriptTerm():
|
|||
<mods:scriptTerm authority="iso15924" type="code">216</mods:scriptTerm>
|
||||
</mods:language>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['language_scriptTerm'] == {'215', '216', '217'}
|
||||
"""
|
||||
)
|
||||
assert d["language_scriptTerm"] == {"215", "216", "217"}
|
||||
|
||||
|
||||
def test_recordInfo():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:recordInfo>
|
||||
<mods:recordIdentifier source="gbv-ppn">PPN610714341</mods:recordIdentifier>
|
||||
</mods:recordInfo>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['recordInfo_recordIdentifier'] == 'PPN610714341'
|
||||
"""
|
||||
)
|
||||
assert d["recordInfo_recordIdentifier"] == "PPN610714341"
|
||||
|
||||
|
||||
def test_accessCondition():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:accessCondition type="use and reproduction">UNKNOWN</mods:accessCondition>
|
||||
</mods:mods>
|
||||
""")
|
||||
assert d['accessCondition-use and reproduction'] == 'UNKNOWN'
|
||||
"""
|
||||
)
|
||||
assert d["accessCondition-use and reproduction"] == "UNKNOWN"
|
||||
|
||||
|
||||
def test_originInfo_no_event_type():
|
||||
with pytest.warns(UserWarning) as ws:
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:originInfo>
|
||||
<mods:place><mods:placeTerm type="text">Berlin</mods:placeTerm></mods:place>
|
||||
</mods:originInfo>
|
||||
</mods:mods>
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
assert d == {} # empty
|
||||
|
||||
assert len(ws) == 1
|
||||
assert ws[0].message.args[0] == 'Filtered {http://www.loc.gov/mods/v3}originInfo element (has no eventType)'
|
||||
assert (
|
||||
ws[0].message.args[0]
|
||||
== "Filtered {http://www.loc.gov/mods/v3}originInfo element (has no eventType)"
|
||||
)
|
||||
|
||||
|
||||
def test_relatedItem():
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:relatedItem type="original">
|
||||
<mods:recordInfo>
|
||||
|
|
@ -139,12 +169,14 @@ def test_relatedItem():
|
|||
</mods:recordInfo>
|
||||
</mods:relatedItem>
|
||||
</mods:mods>
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
assert d['relatedItem-original_recordInfo_recordIdentifier'] == 'PPN167755803'
|
||||
assert d["relatedItem-original_recordInfo_recordIdentifier"] == "PPN167755803"
|
||||
|
||||
# mods:relatedItem may also have source="dnb-ppn" recordIdentifiers:
|
||||
d = dict_fromstring("""
|
||||
d = dict_fromstring(
|
||||
"""
|
||||
<mods:mods xmlns:mods="http://www.loc.gov/mods/v3">
|
||||
<mods:relatedItem type="original">
|
||||
<mods:recordInfo>
|
||||
|
|
@ -152,12 +184,16 @@ def test_relatedItem():
|
|||
</mods:recordInfo>
|
||||
</mods:relatedItem>
|
||||
</mods:mods>
|
||||
""")
|
||||
"""
|
||||
)
|
||||
|
||||
assert d["relatedItem-original_recordInfo_recordIdentifier-dnb-ppn"] == "1236513355"
|
||||
|
||||
assert d['relatedItem-original_recordInfo_recordIdentifier-dnb-ppn'] == '1236513355'
|
||||
|
||||
def test_dtypes(tmp_path):
|
||||
mets_files = [p.absolute().as_posix() for p in (TESTS_DATA_DIR / "mets-mods").glob("*.xml")]
|
||||
mets_files = [
|
||||
p.absolute().as_posix() for p in (TESTS_DATA_DIR / "mets-mods").glob("*.xml")
|
||||
]
|
||||
mods_info_df_parquet = (tmp_path / "test_dtypes_mods_info.parquet").as_posix()
|
||||
page_info_df_parquet = (tmp_path / "test_dtypes_page_info.parquet").as_posix()
|
||||
process(mets_files, mods_info_df_parquet, page_info_df_parquet)
|
||||
|
|
@ -166,7 +202,6 @@ def test_dtypes(tmp_path):
|
|||
|
||||
EXPECTED_TYPES = {
|
||||
# mods_info
|
||||
|
||||
r"mets_file": ("object", ["str"]),
|
||||
r"titleInfo_title": ("object", ["str"]),
|
||||
r"titleInfo_subTitle": ("object", ["str", "NoneType"]),
|
||||
|
|
@ -179,19 +214,16 @@ def test_dtypes(tmp_path):
|
|||
r"typeOfResource": ("object", ["str", "NoneType"]),
|
||||
r"accessCondition-.*": ("object", ["str", "NoneType"]),
|
||||
r"originInfo-.*": ("object", ["str", "NoneType"]),
|
||||
|
||||
r".*-count": ("Int64", None),
|
||||
|
||||
r"genre-.*": ("object", ["ndarray", "NoneType"]),
|
||||
r"subject-.*": ("object", ["ndarray", "NoneType"]),
|
||||
r"language_.*Term": ("object", ["ndarray", "NoneType"]),
|
||||
r"classification-.*": ("object", ["ndarray", "NoneType"]),
|
||||
|
||||
# page_info
|
||||
|
||||
r"fileGrp_.*_file_FLocat_href": ("object", ["str", "NoneType"]),
|
||||
r"structMap-LOGICAL_TYPE_.*": ("boolean", None),
|
||||
}
|
||||
|
||||
def expected_types(c):
|
||||
"""Return the expected types for column c."""
|
||||
for r, types in EXPECTED_TYPES.items():
|
||||
|
|
@ -215,8 +247,9 @@ def test_dtypes(tmp_path):
|
|||
|
||||
if edt == "object":
|
||||
inner_types = set(type(v).__name__ for v in df[c])
|
||||
assert all(it in einner_types for it in inner_types), \
|
||||
f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})"
|
||||
assert all(
|
||||
it in einner_types for it in inner_types
|
||||
), f"Unexpected inner types {inner_types} for column {c} (expected {einner_types})"
|
||||
|
||||
check_types(mods_info_df)
|
||||
check_types(page_info_df)
|
||||
check_types(page_info_df)
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ TESTS_DATA_DIR = Path(__file__).parent / "data"
|
|||
|
||||
|
||||
def removeprefix(s, prefix):
|
||||
if sys.version_info < (3,9):
|
||||
return s[len(prefix):] if s.startswith(prefix) else s
|
||||
if sys.version_info < (3, 9):
|
||||
return s[len(prefix) :] if s.startswith(prefix) else s
|
||||
else:
|
||||
return s.removeprefix(prefix)
|
||||
|
||||
|
|
@ -26,20 +26,32 @@ def test_page_info():
|
|||
assert all(p["ppn"] == "PPN821507109" for p in page_info)
|
||||
|
||||
# Look closer at an interesting page
|
||||
from pprint import pprint; pprint(page_info[0])
|
||||
from pprint import pprint
|
||||
|
||||
pprint(page_info[0])
|
||||
page_info_page = next(p for p in page_info if p["ID"] == "PHYS_0005")
|
||||
|
||||
assert page_info_page["fileGrp_PRESENTATION_file_FLocat_href"] == "file:///goobi/tiff001/sbb/PPN821507109/00000005.tif"
|
||||
assert (
|
||||
page_info_page["fileGrp_PRESENTATION_file_FLocat_href"]
|
||||
== "file:///goobi/tiff001/sbb/PPN821507109/00000005.tif"
|
||||
)
|
||||
|
||||
# This is a title page with an illustration, check that we correctly got this info from the
|
||||
# structMap.
|
||||
struct_types = sorted(removeprefix(k, "structMap-LOGICAL_TYPE_") for k, v in page_info_page.items() if k.startswith("structMap-LOGICAL_TYPE_") and v == 1)
|
||||
struct_types = sorted(
|
||||
removeprefix(k, "structMap-LOGICAL_TYPE_")
|
||||
for k, v in page_info_page.items()
|
||||
if k.startswith("structMap-LOGICAL_TYPE_") and v == 1
|
||||
)
|
||||
assert struct_types == ["illustration", "monograph", "title_page"]
|
||||
|
||||
|
||||
def test_page_info_multivolume_work():
|
||||
"""Test creation of page_info for multivolume_work"""
|
||||
mets = ET.parse(TESTS_DATA_DIR / "mets-mods" / "PPN717884805-multivolume_work-no-structMap-PHYSICAL.xml")
|
||||
mets = ET.parse(
|
||||
TESTS_DATA_DIR
|
||||
/ "mets-mods"
|
||||
/ "PPN717884805-multivolume_work-no-structMap-PHYSICAL.xml"
|
||||
)
|
||||
page_info = pages_to_dict(mets)
|
||||
assert page_info == []
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue