ocrd-galley/ppn2ocr

#!/usr/bin/env python3
"""Get OCR results as a OCR-D workspace for a given PPN"""
import os
import requests
import sys
import lxml.etree as ET
import re
import subprocess
from copy import deepcopy


XMLNS = {
   'mets': 'http://www.loc.gov/METS/',
   'xlink': 'http://www.w3.org/1999/xlink'
}
API_URL = 'https://digital.staatsbibliothek-berlin.de/oai'
IDENTIFIER_TEMPLATE = 'oai:digital.staatsbibliothek-berlin.de:%s'


for prefix, uri in XMLNS.items():
    ET.register_namespace(prefix, uri)


# XXX
#  show_help() {
#    cat <<-EOH
#  Usage: $0 PPN77164308X
#
#    Get OCR results as a OCR-D workspace for a given PPN
#
#  Options:
#    --help                          Show this message and exit.
#  EOH
#  }


def oai_mets(ppn):
    """Retrieve METS metadata for a given PPN."""

    params = {
        'verb': 'GetRecord',
        'metadataPrefix': 'mets',
        'identifier': IDENTIFIER_TEMPLATE % ppn
    }

    s = requests.Session()
    r = s.get(API_URL, params=params)
    mets = ET.XML(r.content).find(f".//{{{XMLNS['mets']}}}mets")
    mets = ET.ElementTree(mets)

    return mets


def iiif_url_for_dms_url(dms_url, ppn, size):
    """
    Construct an IIIF URL from a dms URL.

    This function exists to contain the hack of rewriting the URL to get IIIF.
    """
    if ppn not in dms_url:
        raise ValueError(f"Unexpected URL {dms_url}")
    m = re.search(r'/dms/.*/([0-9]+)\.jpg$', dms_url)
    if m:
        page_num = m.group(1)
    else:
        raise ValueError(f"Unexpected URL {dms_url}")
    iiif_identifier = f'{ppn}-{page_num}'
    iiif_url = f'https://content.staatsbibliothek-berlin.de/dc/{iiif_identifier}/full/{size}/0/default.jpg'

    return iiif_url


def remove_file_grp(mets, use):
    for bad_fileid in mets.xpath(f'//mets:fileGrp[@USE="{use}"]/mets:file/@ID', namespaces=XMLNS):
        for bad in mets.xpath(f'//mets:fptr[@FILEID="{bad_fileid}"]', namespaces=XMLNS):
            bad.getparent().remove(bad)
    for bad in mets.xpath(f'//mets:fileGrp[@USE="{use}"]', namespaces=XMLNS):
        bad.getparent().remove(bad)


def make_workspace(ppn, workspace):
    # Make workspace directory
    os.mkdir(workspace)
    os.chdir(workspace)

    mets = oai_mets(ppn)

    # XXX
    # Delete PRESENTATION file group
    # (local file:/// links, not handled well by "ocrd workspace")
    remove_file_grp(mets, 'PRESENTATION')


    # Duplicate DEFAULT file group into a new file group BEST
    file_grp_default = mets.find('//mets:fileGrp[@USE="DEFAULT"]', namespaces=XMLNS)
    file_grp_best = deepcopy(file_grp_default)

    file_grp_best.attrib['USE'] = 'BEST'
    for f in file_grp_best.findall('./mets:file', namespaces=XMLNS):
        old_id = f.attrib['ID']
        new_id = re.sub('DEFAULT', 'BEST', old_id)
        f.attrib['ID'] = new_id

        for fptr in mets.findall(f'//mets:fptr[@FILEID="{old_id}"]', namespaces=XMLNS):
            new_fptr = deepcopy(fptr)
            new_fptr.attrib['FILEID'] = new_id
            fptr.getparent().append(new_fptr)

        # XXX Need to fumble around with the URL for now
        flocat = f.find(f".//{{{XMLNS['mets']}}}FLocat")
        old_url = flocat.attrib[f"{{{XMLNS['xlink']}}}href"]
        url_iiif_full = iiif_url_for_dms_url(old_url, ppn, 'full')
        flocat.attrib[f"{{{XMLNS['xlink']}}}href"] = url_iiif_full

    mets.find('//mets:fileSec', namespaces=XMLNS).append(file_grp_best)


    # Write mets.xml
    mets.write('mets.xml', pretty_print=True)

    # TODO
    # Validate workspace
    #ocrd workspace validate mets.xml | grep -v "<notice>Won't download remote image"

    # XXX
    # Fix 'file:/' URLs to 'file:///'
    #sed -i 's#file:/\([^/]\)#file:///\1#' mets.xml

    # Patch mets.xml to use our NFS mount
    #sed -i 's#file:///goobi/tiff001/sbb/#file:///srv/digisam_images/sbb/#g' mets.xml


# Command line parameters
# XXX
ppn = sys.argv[1]
#  OPTS=`getopt -o h --long help -- "$@"`
#  eval set -- "$OPTS"
#  while true; do
#      case "$1" in
#          -h|--help) show_help; exit; shift;;
#          --) shift; break;;
#          *) break;;
#      esac
#  done
#  if [ -z "$1" ]; then show_help; exit; fi
#  ppn=$1


# From here, the magic happens
self_dir = os.path.realpath(os.path.dirname(sys.argv[0]))
make_workspace(ppn, ppn)
subprocess.run([
   os.path.join(self_dir, 'run-docker-hub'),
   '-I', 'BEST',
   '--skip-validation'
])


# TODO

# my_ocrd_workflow
# ----------------
# * Need option to add volumes e.g. /srv/digisam_images

# File bugs in OCR-D
# ------------------
# * <error>PAGE-XML OCR-D-IMG-BINPAGE/OCR-D-IMG-BINPAGE_0001.xml : imageFilename '/srv/digisam_images/sbb/PPN719671574/00000001.tif' not found in METS</error>
#   -> had to use relative file names
# * Should be able to disable in workspace validate   <notice>Won't download remote image <http://content.staatsbibliothek-berlin.de/dms/PPN719671574/800/0/00000335.jpg></notice>


# sbb_textline_detector
# ---------------------
# * sbb_textline_detector slow
#   -> Support loading the models once so the OCR-D processor can profit from  processing multiple pages
# * Check what happens with the skewed textlines in SEG_LINE_0019