From 968572168e7816c7740c0ff61faa3385c88ee605 Mon Sep 17 00:00:00 2001
From: Mike Gerber <mike.gerber@sbb.spk-berlin.de>
Date: Thu, 23 Nov 2023 15:00:06 +0100
Subject: [PATCH] =?UTF-8?q?=F0=9F=A7=B9=20Extract=20a=20function=20to=20co?=
 =?UTF-8?q?nvert=20list[dict]=20to=20a=20DataFrame?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 qurator/mods4pandas/lib.py         | 26 +++++++++++++++++++++++++-
 qurator/mods4pandas/mods4pandas.py | 11 ++---------
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/qurator/mods4pandas/lib.py b/qurator/mods4pandas/lib.py
index f3c6078..cb44656 100644
--- a/qurator/mods4pandas/lib.py
+++ b/qurator/mods4pandas/lib.py
@@ -1,8 +1,9 @@
 from itertools import groupby
 import re
 import warnings
-from typing import List, Sequence, MutableMapping
+from typing import List, Sequence, MutableMapping, Dict
 
+import pandas as pd
 import numpy as np
 from lxml import etree as ET
 
@@ -298,3 +299,26 @@ def flatten(d: MutableMapping, parent='', separator='_'):
 
     return dict(items)
 
+
+def dicts_to_df(data_list: List[Dict], *, index_column: str) -> pd.DataFrame:
+    """
+    Convert the given list of dicts to a Pandas DataFrame.
+
+    The keys of the dicts make the columns.
+    """
+
+    # Build columns from keys
+    columns = []
+    for m in data_list:
+        for c in m.keys():
+            if c not in columns:
+                columns.append(c)
+
+    # Build data table
+    data = [[m.get(c) for c in columns] for m in data_list]
+
+    # Build index
+    index = [m[index_column] for m in data_list]
+
+    df = pd.DataFrame(data=data, index=index, columns=columns)
+    return df
diff --git a/qurator/mods4pandas/mods4pandas.py b/qurator/mods4pandas/mods4pandas.py
index 94bddd3..f651556 100755
--- a/qurator/mods4pandas/mods4pandas.py
+++ b/qurator/mods4pandas/mods4pandas.py
@@ -14,7 +14,7 @@ import click
 import pandas as pd
 from tqdm import tqdm
 
-from .lib import sorted_groupby, TagGroup, ns, flatten
+from .lib import sorted_groupby, TagGroup, ns, flatten, dicts_to_df
 
 
 
@@ -404,14 +404,7 @@ def process(mets_files: List[str], output_file: str, output_csv: str, output_xls
                 #import traceback; traceback.print_exc()
 
     # Convert the mods_info List[Dict] to a pandas DataFrame
-    columns = []
-    for m in mods_info:
-        for c in m.keys():
-            if c not in columns:
-                columns.append(c)
-    data = [[m.get(c) for c in columns] for m in mods_info]
-    index = [m['recordInfo_recordIdentifier'] for m in mods_info]  # PPN
-    mods_info_df = pd.DataFrame(data=data, index=index, columns=columns)
+    mods_info_df = dicts_to_df(mods_info, index_column="recordInfo_recordIdentifier")
 
     # Pickle the DataFrame
     logger.info('Writing DataFrame to {}'.format(output_file))