Introduce version pinning and license checcking

Introduces version pinning via pip-tools for reproducible builds. Introduces license checking via pip-licenses and CircleCI. Licenses are only checked for new builds and when requirements.txt changes. The list of allowed licenses is kept in a separate file to be able to distinguish changes in CI-Configuration/Tools and license list changes.
4 years ago · 15e584f0ab
parent 1778b36a9a
commit 15e584f0ab
6 changed files with 296 additions and 19 deletions
--- a/.allowed-licenses
+++ b/.allowed-licenses
@ -0,0 +1,34 @@
+# Allowed licenses for this project, one license per line
+
+# Variants of the Apache Software License
+Apache
+Apache License 2.0
+Apache Software License
+Apache Software License, BSD License
+
+# Variants of the BSD License
+BSD
+BSD License
+
+# Variants of the LGPL
+GNU Library or Lesser General Public License (LGPL)
+LGPL
+
+# Variants of the HPND License
+Historical Permission Notice and Disclaimer (HPND)
+
+# Variants of the MIT License
+MIT
+MIT License
+MIT License, Mozilla Public License 2.0 (MPL 2.0)
+
+# Variants of the MLP
+MPL
+MPL 2.0
+Mozilla Public License 2.0 (MPL 2.0)
+
+# Public Domain Variants
+Public Domain
+
+# Variants of the Python Software Foundation License
+Python Software Foundation License
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -9,10 +9,41 @@ jobs:
      - image: circleci/python:<< parameters.python-version >>
    steps:
      - checkout
-      - run: pip3 install --upgrade pip
-      - run: pip3 install -r requirements.txt
-      - run: pip3 install pytest
+      - &install
+        run:
+          name: Install packages
+          command: |
+            pip3 install --upgrade pip
+            pip3 install -r requirements.txt
+      - run:
+          name: Install specific packages
+          command: pip3 install pytest
      - run: pytest
+  license-scan:
+    docker:
+      - image: circleci/python:3.9
+    steps:
+      - checkout
+      - run:  # Only run license checks if we have changes in requirements.txt
+          name: Check for changes
+          command: |
+            if [ -z "<< pipeline.git.base_revision >>" ]; then
+              echo "No previous build, run license check by default."
+            elif git diff-tree --no-commit-id --name-only -r << pipeline.git.revision >> << pipeline.git.base_revision >> | grep requirements.txt ; then
+              echo "Changes in requirements.txt, run license check."
+            else
+              echo "No relevant changes found, skip running license check."
+              circleci-agent step halt
+            fi
+      - *install
+      - run:
+          name: Install specific packages
+          command: pip3 install pip-licenses
+      - run:  # Read allowed licenses from file to ";"-separated string while removing empty lines and comments
+          name: License check
+          command: |
+            ALLOWED=$(sed -e '/^#.*$/d' -e '/^\s*$/d' .allowed-licenses | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/;/g'); echo $ALLOWED
+            pip-licenses --from=mixed --allow-only="${ALLOWED}" --summary

 workflows:
  all-tests:
@ -21,3 +52,9 @@ workflows:
          matrix:
            parameters:
              python-version: ["3.5", "3.6", "3.7", "3.8", "3.9"]
+  license-scan:
+    jobs:
+      - license-scan:
+          filters:
+            branches:
+              only: master
--- a/requirements-dev.in
+++ b/requirements-dev.in
@ -0,0 +1,7 @@
+-c requirements.txt
+pip-tools
+pytest
+pytest-flake8
+pytest-cov
+pytest-mypy
+black
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -1,5 +1,88 @@
-pytest
-pytest-flake8
-pytest-cov
-pytest-mypy
-black
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile '.\requirements-dev.in'
+#
+appdirs==1.4.4
+    # via black
+atomicwrites==1.4.0
+    # via
+    #   -c .\requirements.txt
+    #   pytest
+attrs==20.3.0
+    # via
+    #   -c .\requirements.txt
+    #   pytest
+    #   pytest-mypy
+black==21.5b0
+    # via -r .\requirements-dev.in
+click==7.1.2
+    # via
+    #   -c .\requirements.txt
+    #   black
+    #   pip-tools
+colorama==0.4.4
+    # via
+    #   -c .\requirements.txt
+    #   pytest
+coverage==5.5
+    # via pytest-cov
+filelock==3.0.12
+    # via pytest-mypy
+flake8==3.9.1
+    # via pytest-flake8
+iniconfig==1.1.1
+    # via pytest
+mccabe==0.6.1
+    # via flake8
+mypy-extensions==0.4.3
+    # via
+    #   black
+    #   mypy
+mypy==0.812
+    # via pytest-mypy
+packaging==20.9
+    # via pytest
+pathspec==0.8.1
+    # via black
+pep517==0.10.0
+    # via pip-tools
+pip-tools==6.1.0
+    # via -r .\requirements-dev.in
+pluggy==0.13.1
+    # via pytest
+py==1.10.0
+    # via pytest
+pycodestyle==2.7.0
+    # via flake8
+pyflakes==2.3.1
+    # via flake8
+pyparsing==2.4.7
+    # via packaging
+pytest-cov==2.11.1
+    # via -r .\requirements-dev.in
+pytest-flake8==1.0.7
+    # via -r .\requirements-dev.in
+pytest-mypy==0.8.1
+    # via -r .\requirements-dev.in
+pytest==6.2.4
+    # via
+    #   -r .\requirements-dev.in
+    #   pytest-cov
+    #   pytest-flake8
+    #   pytest-mypy
+regex==2021.4.4
+    # via black
+toml==0.10.2
+    # via
+    #   black
+    #   pep517
+    #   pytest
+typed-ast==1.4.3
+    # via mypy
+typing-extensions==3.10.0.0
+    # via mypy
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
--- a/requirements.in
+++ b/requirements.in
@ -0,0 +1,11 @@
+click
+jinja2
+lxml
+uniseg
+numpy
+colorama
+MarkupSafe
+ocrd >= 2.20.1
+attrs
+multimethod == 1.3  # latest version to officially support Python 3.5
+tqdm
--- a/requirements.txt
+++ b/requirements.txt
@ -1,11 +1,116 @@
-click
-jinja2
-lxml
-uniseg
-numpy
-colorama
-MarkupSafe
-ocrd >= 2.20.1
-attrs
-multimethod == 1.3  # latest version to officially support Python 3.5
-tqdm
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile '.\requirements.in'
+#
+atomicwrites==1.4.0
+    # via ocrd-utils
+attrs==20.3.0
+    # via
+    #   -r .\requirements.in
+    #   jsonschema
+bagit-profile==1.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+bagit==1.8.1
+    # via
+    #   bagit-profile
+    #   ocrd
+    #   ocrd-validators
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   -r .\requirements.in
+    #   flask
+    #   ocrd
+    #   ocrd-validators
+colorama==0.4.4
+    # via -r .\requirements.in
+deprecated==1.2.0
+    # via ocrd
+flask==1.1.2
+    # via ocrd
+idna==2.10
+    # via requests
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via
+    #   -r .\requirements.in
+    #   flask
+jsonschema==3.2.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+lxml==4.6.3
+    # via
+    #   -r .\requirements.in
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+markupsafe==1.1.1
+    # via
+    #   -r .\requirements.in
+    #   jinja2
+multimethod==1.3
+    # via -r .\requirements.in
+numpy==1.20.2
+    # via
+    #   -r .\requirements.in
+    #   ocrd-utils
+    #   opencv-python-headless
+ocrd-modelfactory==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+ocrd-models==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-validators
+ocrd-utils==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+    #   ocrd-validators
+ocrd-validators==2.24.0
+    # via ocrd
+ocrd==2.24.0
+    # via -r .\requirements.in
+opencv-python-headless==4.5.1.48
+    # via ocrd
+pillow==8.2.0
+    # via ocrd-utils
+pyrsistent==0.17.3
+    # via jsonschema
+pyyaml==5.4.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+requests==2.25.1
+    # via
+    #   bagit-profile
+    #   ocrd
+shapely==1.7.1
+    # via ocrd-validators
+six==1.15.0
+    # via jsonschema
+tqdm==4.60.0
+    # via -r .\requirements.in
+uniseg==0.7.1.post2
+    # via -r .\requirements.in
+urllib3==1.26.4
+    # via requests
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools