Introduce version pinning and license checcking

Introduces version pinning via pip-tools for reproducible builds.
Introduces license checking via pip-licenses and CircleCI.

Licenses are only checked for new builds and when requirements.txt changes.

The list of allowed licenses is kept in a separate file to be able to
distinguish changes in CI-Configuration/Tools and license list changes.

.allowed-licenses

@@ -0,0 +1,34 @@
+# Allowed licenses for this project, one license per line
+
+# Variants of the Apache Software License
+Apache
+Apache License 2.0
+Apache Software License
+Apache Software License, BSD License
+
+# Variants of the BSD License
+BSD
+BSD License
+
+# Variants of the LGPL
+GNU Library or Lesser General Public License (LGPL)
+LGPL
+
+# Variants of the HPND License
+Historical Permission Notice and Disclaimer (HPND)
+
+# Variants of the MIT License
+MIT
+MIT License
+MIT License, Mozilla Public License 2.0 (MPL 2.0)
+
+# Variants of the MLP
+MPL
+MPL 2.0
+Mozilla Public License 2.0 (MPL 2.0)
+
+# Public Domain Variants
+Public Domain
+
+# Variants of the Python Software Foundation License
+Python Software Foundation License

.circleci/config.yml

@@ -9,10 +9,41 @@ jobs:
       - image: circleci/python:<< parameters.python-version >>
     steps:
       - checkout
-      - run: pip3 install --upgrade pip
+      - &install
-      - run: pip3 install -r requirements.txt
+        run:
-      - run: pip3 install pytest
+          name: Install packages
+          command: |
+            pip3 install --upgrade pip
+            pip3 install -r requirements.txt
+      - run:
+          name: Install specific packages
+          command: pip3 install pytest
       - run: pytest
+  license-scan:
+    docker:
+      - image: circleci/python:3.9
+    steps:
+      - checkout
+      - run:  # Only run license checks if we have changes in requirements.txt
+          name: Check for changes
+          command: |
+            if [ -z "<< pipeline.git.base_revision >>" ]; then
+              echo "No previous build, run license check by default."
+            elif git diff-tree --no-commit-id --name-only -r << pipeline.git.revision >> << pipeline.git.base_revision >> | grep requirements.txt ; then
+              echo "Changes in requirements.txt, run license check."
+            else
+              echo "No relevant changes found, skip running license check."
+              circleci-agent step halt
+            fi
+      - *install
+      - run:
+          name: Install specific packages
+          command: pip3 install pip-licenses
+      - run:  # Read allowed licenses from file to ";"-separated string while removing empty lines and comments
+          name: License check
+          command: |
+            ALLOWED=$(sed -e '/^#.*$/d' -e '/^\s*$/d' .allowed-licenses | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/;/g'); echo $ALLOWED
+            pip-licenses --from=mixed --allow-only="${ALLOWED}" --summary
 
 workflows:
   all-tests:
@@ -21,3 +52,9 @@ workflows:
           matrix:
             parameters:
               python-version: ["3.5", "3.6", "3.7", "3.8", "3.9"]
+  license-scan:
+    jobs:
+      - license-scan:
+          filters:
+            branches:
+              only: master

requirements-dev.in

@@ -0,0 +1,7 @@
+-c requirements.txt
+pip-tools
+pytest
+pytest-flake8
+pytest-cov
+pytest-mypy
+black

requirements-dev.txt

@@ -1,5 +1,88 @@
-pytest
+#
-pytest-flake8
+# This file is autogenerated by pip-compile
-pytest-cov
+# To update, run:
-pytest-mypy
+#
-black
+#    pip-compile '.\requirements-dev.in'
+#
+appdirs==1.4.4
+    # via black
+atomicwrites==1.4.0
+    # via
+    #   -c .\requirements.txt
+    #   pytest
+attrs==20.3.0
+    # via
+    #   -c .\requirements.txt
+    #   pytest
+    #   pytest-mypy
+black==21.5b0
+    # via -r .\requirements-dev.in
+click==7.1.2
+    # via
+    #   -c .\requirements.txt
+    #   black
+    #   pip-tools
+colorama==0.4.4
+    # via
+    #   -c .\requirements.txt
+    #   pytest
+coverage==5.5
+    # via pytest-cov
+filelock==3.0.12
+    # via pytest-mypy
+flake8==3.9.1
+    # via pytest-flake8
+iniconfig==1.1.1
+    # via pytest
+mccabe==0.6.1
+    # via flake8
+mypy-extensions==0.4.3
+    # via
+    #   black
+    #   mypy
+mypy==0.812
+    # via pytest-mypy
+packaging==20.9
+    # via pytest
+pathspec==0.8.1
+    # via black
+pep517==0.10.0
+    # via pip-tools
+pip-tools==6.1.0
+    # via -r .\requirements-dev.in
+pluggy==0.13.1
+    # via pytest
+py==1.10.0
+    # via pytest
+pycodestyle==2.7.0
+    # via flake8
+pyflakes==2.3.1
+    # via flake8
+pyparsing==2.4.7
+    # via packaging
+pytest-cov==2.11.1
+    # via -r .\requirements-dev.in
+pytest-flake8==1.0.7
+    # via -r .\requirements-dev.in
+pytest-mypy==0.8.1
+    # via -r .\requirements-dev.in
+pytest==6.2.4
+    # via
+    #   -r .\requirements-dev.in
+    #   pytest-cov
+    #   pytest-flake8
+    #   pytest-mypy
+regex==2021.4.4
+    # via black
+toml==0.10.2
+    # via
+    #   black
+    #   pep517
+    #   pytest
+typed-ast==1.4.3
+    # via mypy
+typing-extensions==3.10.0.0
+    # via mypy
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip

requirements.in

@@ -0,0 +1,11 @@
+click
+jinja2
+lxml
+uniseg
+numpy
+colorama
+MarkupSafe
+ocrd >= 2.20.1
+attrs
+multimethod == 1.3  # latest version to officially support Python 3.5
+tqdm

requirements.txt

@@ -1,11 +1,116 @@
-click
+#
-jinja2
+# This file is autogenerated by pip-compile
-lxml
+# To update, run:
-uniseg
+#
-numpy
+#    pip-compile '.\requirements.in'
-colorama
+#
-MarkupSafe
+atomicwrites==1.4.0
-ocrd >= 2.20.1
+    # via ocrd-utils
-attrs
+attrs==20.3.0
-multimethod == 1.3  # latest version to officially support Python 3.5
+    # via
-tqdm
+    #   -r .\requirements.in
+    #   jsonschema
+bagit-profile==1.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+bagit==1.8.1
+    # via
+    #   bagit-profile
+    #   ocrd
+    #   ocrd-validators
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   -r .\requirements.in
+    #   flask
+    #   ocrd
+    #   ocrd-validators
+colorama==0.4.4
+    # via -r .\requirements.in
+deprecated==1.2.0
+    # via ocrd
+flask==1.1.2
+    # via ocrd
+idna==2.10
+    # via requests
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via
+    #   -r .\requirements.in
+    #   flask
+jsonschema==3.2.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+lxml==4.6.3
+    # via
+    #   -r .\requirements.in
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+markupsafe==1.1.1
+    # via
+    #   -r .\requirements.in
+    #   jinja2
+multimethod==1.3
+    # via -r .\requirements.in
+numpy==1.20.2
+    # via
+    #   -r .\requirements.in
+    #   ocrd-utils
+    #   opencv-python-headless
+ocrd-modelfactory==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+ocrd-models==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-validators
+ocrd-utils==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+    #   ocrd-validators
+ocrd-validators==2.24.0
+    # via ocrd
+ocrd==2.24.0
+    # via -r .\requirements.in
+opencv-python-headless==4.5.1.48
+    # via ocrd
+pillow==8.2.0
+    # via ocrd-utils
+pyrsistent==0.17.3
+    # via jsonschema
+pyyaml==5.4.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+requests==2.25.1
+    # via
+    #   bagit-profile
+    #   ocrd
+shapely==1.7.1
+    # via ocrd-validators
+six==1.15.0
+    # via jsonschema
+tqdm==4.60.0
+    # via -r .\requirements.in
+uniseg==0.7.1.post2
+    # via -r .\requirements.in
+urllib3==1.26.4
+    # via requests
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools