import json import os from collections import Counter from functools import partial from typing import Any, Callable, Dict, List, Tuple import click from jinja2 import Environment, FileSystemLoader from markupsafe import escape from ocrd_utils import initLogging from .align import seq_align from .config import Config from .extracted_text import ExtractedText from .metrics import ( bag_of_chars_accuracy, bag_of_words_accuracy, character_accuracy, word_accuracy, ) from .normalize import chars_normalized, words_normalized from .ocr_files import extract def gen_count_report( gt_text: ExtractedText, ocr_text: ExtractedText, split_fun: Callable[[str], Counter] ) -> List[Tuple[str, int, int]]: gt_counter = Counter(split_fun(gt_text.text)) ocr_counter = Counter(split_fun(ocr_text.text)) return [ ("".join(key), gt_counter[key], ocr_counter[key]) for key in sorted({*gt_counter.keys(), *ocr_counter.keys()}) ] def gen_diff_report( gt_in: ExtractedText, ocr_in: ExtractedText, css_prefix: str = "c", joiner: str = "", none: str = "·", split_fun=chars_normalized, ) -> Tuple[str, str]: gtx = "" ocrx = "" def format_thing(t, css_classes=None, id_=None): if t is None: html_t = none css_classes += " ellipsis" elif t == "\n": html_t = "
" else: html_t = escape(t) html_custom_attrs = "" # Set Bootstrap tooltip to the segment id if id_: html_custom_attrs += 'data-toggle="tooltip" title="{}"'.format(id_) if css_classes: return f'{html_t}' else: return f"{html_t}" gt_things = split_fun(gt_in.text) ocr_things = split_fun(ocr_in.text) g_pos = 0 o_pos = 0 for k, (g, o) in enumerate(seq_align(gt_things, ocr_things)): css_classes = None gt_id = None ocr_id = None if g != o: css_classes = "{css_prefix}diff{k} diff".format(css_prefix=css_prefix, k=k) gt_id = gt_in.segment_id_for_pos(g_pos) if g is not None else None ocr_id = ocr_in.segment_id_for_pos(o_pos) if o is not None else None # Deletions and inserts only produce one id + None, UI must # support this, i.e. display for the one id produced gtx += joiner + format_thing(g, css_classes, gt_id) ocrx += joiner + format_thing(o, css_classes, ocr_id) if g is not None: g_pos += len(g) if o is not None: o_pos += len(o) return gtx, ocrx def generate_html_report( gt: str, ocr: str, gt_text: ExtractedText, ocr_text: ExtractedText, report_prefix: str, metrics_results: Dict, ): metric_dict: Dict[str, Callable] = { "character_accuracy": partial( gen_diff_report, css_prefix="c", joiner="", none="·", split_fun=chars_normalized, ), "word_accuracy": partial( gen_diff_report, css_prefix="w", joiner=" ", none="⋯", split_fun=words_normalized, ), "bag_of_chars_accuracy": partial(gen_count_report, split_fun=chars_normalized), "bag_of_words_accuracy": partial(gen_count_report, split_fun=words_normalized), } metrics_reports = {} for metric in metrics_results.keys(): if metric not in metric_dict.keys(): raise ValueError(f"Unknown metric '{metric}'.") metrics_reports[metric] = metric_dict[metric](gt_text, ocr_text) env = Environment( loader=FileSystemLoader( os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates") ) ) report_suffix = ".html" template_fn = "report" + report_suffix + ".j2" out_fn = report_prefix + report_suffix template = env.get_template(template_fn) template.stream( gt=gt, ocr=ocr, metrics_reports=metrics_reports, metrics_results=metrics_results, ).dump(out_fn) def generate_json_report(gt: str, ocr: str, report_prefix: str, metrics_results: Dict): json_dict: Dict[str, Any] = {"gt": gt, "ocr": ocr} for result in metrics_results.values(): json_dict[result.metric] = { key: value for key, value in result.get_dict().items() if key != "metric" } with open(f"{report_prefix}.json", "w") as fp: json.dump(json_dict, fp) def process( gt, ocr, report_prefix, *, html=True, metrics="cer,wer", textequiv_level="region" ): """Check OCR result against GT. The @click decorators change the signature of the decorated functions, so we keep this undecorated version and use Click on a wrapper. """ gt_text = extract(gt, textequiv_level=textequiv_level) ocr_text = extract(ocr, textequiv_level=textequiv_level) metrics_results = {} if metrics: metric_dict = { "ca": character_accuracy, "cer": character_accuracy, "wa": word_accuracy, "wer": word_accuracy, "boc": bag_of_chars_accuracy, "bow": bag_of_words_accuracy, } for metric in metrics.split(","): metric = metric.strip() if metric not in metric_dict.keys(): raise ValueError(f"Unknown metric '{metric}'.") result = metric_dict[metric](gt_text.text, ocr_text.text) metrics_results[result.metric] = result generate_json_report(gt, ocr, report_prefix, metrics_results) if html: generate_html_report(gt, ocr, gt_text, ocr_text, report_prefix, metrics_results) @click.command() @click.argument("gt", type=click.Path(exists=True)) @click.argument("ocr", type=click.Path(exists=True)) @click.argument("report_prefix", type=click.Path(), default="report") @click.option("--html", default=True, is_flag=True, help="Enable/disable html report.") @click.option( "--metrics", default="cer,wer", help="Enable different metrics like cer, wer, boc and bow.", ) @click.option( "--textequiv-level", default="region", help="PAGE TextEquiv level to extract text from", metavar="LEVEL", ) @click.option("--progress", default=False, is_flag=True, help="Show progress bar") def main(gt, ocr, report_prefix, html, metrics, textequiv_level, progress): """ Compare the PAGE/ALTO/text document GT against the document OCR. dinglehopper detects if GT/OCR are ALTO or PAGE XML documents to extract their text and falls back to plain text if no ALTO or PAGE is detected. The files GT and OCR are usually a ground truth document and the result of an OCR software, but you may use dinglehopper to compare two OCR results. In that case, use --metrics='' to disable the then meaningless metrics and also change the color scheme from green/red to blue. The comparison report will be written to $REPORT_PREFIX.{html,json}, where $REPORT_PREFIX defaults to "report". Depending on your configuration the reports include the character error rate (CA|CER), the word error rate (WA|WER), the bag of chars accuracy (BoC), and the bag of words accuracy (BoW). The metrics can be chosen via a comma separated combination of their acronyms like "--metrics=ca,wer,boc,bow". The html report can be enabled/disabled using --html / --no-html. By default, the text of PAGE files is extracted on 'region' level. You may use "--textequiv-level line" to extract from the level of TextLine tags. """ initLogging() Config.progress = progress process( gt, ocr, report_prefix, html=html, metrics=metrics, textequiv_level=textequiv_level, ) if __name__ == "__main__": main()