Test with version specific requirement files

.circleci/config.yml

@@ -1,12 +1,15 @@
 version: 2.1
 
+x-environment: &environment
+  parameters:
+    python-version:
+      type: string
+  docker:
+    - image: circleci/python:<< parameters.python-version >>
+
 jobs:
   test:
-    parameters:
+    <<: *environment
-      python-version:
-        type: string
-    docker:
-      - image: circleci/python:<< parameters.python-version >>
     steps:
       - checkout
       - &install
@@ -14,14 +17,13 @@ jobs:
           name: Install packages
           command: |
             pip3 install --upgrade pip
-            pip3 install -r requirements.txt
+            pip3 install -r py<< parameters.python-version >>-requirements.txt
       - run:
           name: Install specific packages
           command: pip3 install pytest
       - run: pytest
   license-scan:
-    docker:
+    <<: *environment
-      - image: circleci/python:3.9
     steps:
       - checkout
       - run:  # Only run license checks if we have changes in requirements.txt
@@ -29,8 +31,8 @@ jobs:
           command: |
             if [ -z "<< pipeline.git.base_revision >>" ]; then
               echo "No previous build, run license check by default."
-            elif git diff-tree --no-commit-id --name-only -r << pipeline.git.revision >> << pipeline.git.base_revision >> | grep requirements.txt ; then
+            elif git diff-tree --no-commit-id --name-only -r << pipeline.git.revision >> << pipeline.git.base_revision >> | grep py<< parameters.python-version >>-requirements.txt ; then
-              echo "Changes in requirements.txt, run license check."
+              echo "Changes in py<< parameters.python-version >>-requirements.txt, run license check."
             else
               echo "No relevant changes found, skip running license check."
               circleci-agent step halt
@@ -45,16 +47,15 @@ jobs:
             ALLOWED=$(sed -e '/^#.*$/d' -e '/^\s*$/d' .allowed-licenses | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/;/g'); echo $ALLOWED
             pip-licenses --from=mixed --allow-only="${ALLOWED}" --summary
 
+x-version-matrix: &version-matrix
+  matrix:
+    parameters:
+      python-version: ["3.5", "3.6", "3.7", "3.8", "3.9"]
+
 workflows:
   all-tests:
     jobs:
       - test:
-          matrix:
+          <<: *version-matrix
-            parameters:
-              python-version: ["3.5", "3.6", "3.7", "3.8", "3.9"]
-  license-scan:
-    jobs:
       - license-scan:
-          filters:
+          <<: *version-matrix
-            branches:
-              only: master

generate-requirements.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Skript to generate Python version specific requirements files in docker containers
+
+for PY_VERSION in 3.5 3.6 3.7 3.8 3.9; do
+    docker run --rm -it -v "${PWD}:/dinglehopper" python:${PY_VERSION} /bin/bash -c "cd /dinglehopper ; pip install pip-tools ; pip-compile --output-file py${PY_VERSION}-requirements.txt requirements.in"
+done

py3.5-requirements.txt

@@ -0,0 +1,120 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=py3.5-requirements.txt requirements.in
+#
+atomicwrites==1.4.0
+    # via ocrd-utils
+attrs==20.3.0
+    # via
+    #   -r requirements.in
+    #   jsonschema
+bagit-profile==1.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+bagit==1.8.1
+    # via
+    #   bagit-profile
+    #   ocrd
+    #   ocrd-validators
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   -r requirements.in
+    #   flask
+    #   ocrd
+    #   ocrd-validators
+colorama==0.4.4
+    # via -r requirements.in
+deprecated==1.2.0
+    # via ocrd
+flask==1.1.2
+    # via ocrd
+idna==2.10
+    # via requests
+importlib-metadata==2.1.1
+    # via jsonschema
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via
+    #   -r requirements.in
+    #   flask
+jsonschema==3.2.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+lxml==4.6.3
+    # via
+    #   -r requirements.in
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+markupsafe==1.1.1
+    # via
+    #   -r requirements.in
+    #   jinja2
+multimethod==1.3
+    # via -r requirements.in
+numpy==1.18.5
+    # via
+    #   -r requirements.in
+    #   ocrd-utils
+    #   opencv-python-headless
+ocrd-modelfactory==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+ocrd-models==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-validators
+ocrd-utils==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+    #   ocrd-validators
+ocrd-validators==2.24.0
+    # via ocrd
+ocrd==2.24.0
+    # via -r requirements.in
+opencv-python-headless==4.4.0.42
+    # via ocrd
+pillow==7.2.0
+    # via ocrd-utils
+pyrsistent==0.17.3
+    # via jsonschema
+pyyaml==5.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+requests==2.25.1
+    # via
+    #   bagit-profile
+    #   ocrd
+shapely==1.7.1
+    # via ocrd-validators
+six==1.15.0
+    # via jsonschema
+tqdm==4.60.0
+    # via -r requirements.in
+uniseg==0.7.1.post2
+    # via -r requirements.in
+urllib3==1.26.4
+    # via requests
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
+zipp==1.2.0
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

py3.6-requirements.txt

@@ -0,0 +1,122 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=py3.6-requirements.txt requirements.in
+#
+atomicwrites==1.4.0
+    # via ocrd-utils
+attrs==20.3.0
+    # via
+    #   -r requirements.in
+    #   jsonschema
+bagit-profile==1.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+bagit==1.8.1
+    # via
+    #   bagit-profile
+    #   ocrd
+    #   ocrd-validators
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   -r requirements.in
+    #   flask
+    #   ocrd
+    #   ocrd-validators
+colorama==0.4.4
+    # via -r requirements.in
+deprecated==1.2.0
+    # via ocrd
+flask==1.1.2
+    # via ocrd
+idna==2.10
+    # via requests
+importlib-metadata==4.0.1
+    # via jsonschema
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via
+    #   -r requirements.in
+    #   flask
+jsonschema==3.2.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+lxml==4.6.3
+    # via
+    #   -r requirements.in
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+markupsafe==1.1.1
+    # via
+    #   -r requirements.in
+    #   jinja2
+multimethod==1.3
+    # via -r requirements.in
+numpy==1.19.5
+    # via
+    #   -r requirements.in
+    #   ocrd-utils
+    #   opencv-python-headless
+ocrd-modelfactory==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+ocrd-models==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-validators
+ocrd-utils==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+    #   ocrd-validators
+ocrd-validators==2.24.0
+    # via ocrd
+ocrd==2.24.0
+    # via -r requirements.in
+opencv-python-headless==4.5.1.48
+    # via ocrd
+pillow==8.2.0
+    # via ocrd-utils
+pyrsistent==0.17.3
+    # via jsonschema
+pyyaml==5.4.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+requests==2.25.1
+    # via
+    #   bagit-profile
+    #   ocrd
+shapely==1.7.1
+    # via ocrd-validators
+six==1.15.0
+    # via jsonschema
+tqdm==4.60.0
+    # via -r requirements.in
+typing-extensions==3.10.0.0
+    # via importlib-metadata
+uniseg==0.7.1.post2
+    # via -r requirements.in
+urllib3==1.26.4
+    # via requests
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
+zipp==3.4.1
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

py3.7-requirements.txt

@@ -0,0 +1,122 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=py3.7-requirements.txt requirements.in
+#
+atomicwrites==1.4.0
+    # via ocrd-utils
+attrs==20.3.0
+    # via
+    #   -r requirements.in
+    #   jsonschema
+bagit-profile==1.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+bagit==1.8.1
+    # via
+    #   bagit-profile
+    #   ocrd
+    #   ocrd-validators
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   -r requirements.in
+    #   flask
+    #   ocrd
+    #   ocrd-validators
+colorama==0.4.4
+    # via -r requirements.in
+deprecated==1.2.0
+    # via ocrd
+flask==1.1.2
+    # via ocrd
+idna==2.10
+    # via requests
+importlib-metadata==4.0.1
+    # via jsonschema
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via
+    #   -r requirements.in
+    #   flask
+jsonschema==3.2.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+lxml==4.6.3
+    # via
+    #   -r requirements.in
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+markupsafe==1.1.1
+    # via
+    #   -r requirements.in
+    #   jinja2
+multimethod==1.3
+    # via -r requirements.in
+numpy==1.20.2
+    # via
+    #   -r requirements.in
+    #   ocrd-utils
+    #   opencv-python-headless
+ocrd-modelfactory==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+ocrd-models==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-validators
+ocrd-utils==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+    #   ocrd-validators
+ocrd-validators==2.24.0
+    # via ocrd
+ocrd==2.24.0
+    # via -r requirements.in
+opencv-python-headless==4.5.1.48
+    # via ocrd
+pillow==8.2.0
+    # via ocrd-utils
+pyrsistent==0.17.3
+    # via jsonschema
+pyyaml==5.4.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+requests==2.25.1
+    # via
+    #   bagit-profile
+    #   ocrd
+shapely==1.7.1
+    # via ocrd-validators
+six==1.15.0
+    # via jsonschema
+tqdm==4.60.0
+    # via -r requirements.in
+typing-extensions==3.10.0.0
+    # via importlib-metadata
+uniseg==0.7.1.post2
+    # via -r requirements.in
+urllib3==1.26.4
+    # via requests
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
+zipp==3.4.1
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

py3.8-requirements.txt

@@ -2,13 +2,13 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile '.\requirements.in'
+#    pip-compile --output-file=py3.8-requirements.txt requirements.in
 #
 atomicwrites==1.4.0
     # via ocrd-utils
 attrs==20.3.0
     # via
-    #   -r .\requirements.in
+    #   -r requirements.in
     #   jsonschema
 bagit-profile==1.3.1
     # via
@@ -25,12 +25,12 @@ chardet==4.0.0
     # via requests
 click==7.1.2
     # via
-    #   -r .\requirements.in
+    #   -r requirements.in
     #   flask
     #   ocrd
     #   ocrd-validators
 colorama==0.4.4
-    # via -r .\requirements.in
+    # via -r requirements.in
 deprecated==1.2.0
     # via ocrd
 flask==1.1.2
@@ -41,7 +41,7 @@ itsdangerous==1.1.0
     # via flask
 jinja2==2.11.3
     # via
-    #   -r .\requirements.in
+    #   -r requirements.in
     #   flask
 jsonschema==3.2.0
     # via
@@ -49,19 +49,19 @@ jsonschema==3.2.0
     #   ocrd-validators
 lxml==4.6.3
     # via
-    #   -r .\requirements.in
+    #   -r requirements.in
     #   ocrd
     #   ocrd-modelfactory
     #   ocrd-models
 markupsafe==1.1.1
     # via
-    #   -r .\requirements.in
+    #   -r requirements.in
     #   jinja2
 multimethod==1.3
-    # via -r .\requirements.in
+    # via -r requirements.in
 numpy==1.20.2
     # via
-    #   -r .\requirements.in
+    #   -r requirements.in
     #   ocrd-utils
     #   opencv-python-headless
 ocrd-modelfactory==2.24.0
@@ -82,7 +82,7 @@ ocrd-utils==2.24.0
 ocrd-validators==2.24.0
     # via ocrd
 ocrd==2.24.0
-    # via -r .\requirements.in
+    # via -r requirements.in
 opencv-python-headless==4.5.1.48
     # via ocrd
 pillow==8.2.0
@@ -102,9 +102,9 @@ shapely==1.7.1
 six==1.15.0
     # via jsonschema
 tqdm==4.60.0
-    # via -r .\requirements.in
+    # via -r requirements.in
 uniseg==0.7.1.post2
-    # via -r .\requirements.in
+    # via -r requirements.in
 urllib3==1.26.4
     # via requests
 werkzeug==1.0.1

py3.9-requirements.txt

@@ -0,0 +1,116 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=py3.9-requirements.txt requirements.in
+#
+atomicwrites==1.4.0
+    # via ocrd-utils
+attrs==20.3.0
+    # via
+    #   -r requirements.in
+    #   jsonschema
+bagit-profile==1.3.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+bagit==1.8.1
+    # via
+    #   bagit-profile
+    #   ocrd
+    #   ocrd-validators
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via
+    #   -r requirements.in
+    #   flask
+    #   ocrd
+    #   ocrd-validators
+colorama==0.4.4
+    # via -r requirements.in
+deprecated==1.2.0
+    # via ocrd
+flask==1.1.2
+    # via ocrd
+idna==2.10
+    # via requests
+itsdangerous==1.1.0
+    # via flask
+jinja2==2.11.3
+    # via
+    #   -r requirements.in
+    #   flask
+jsonschema==3.2.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+lxml==4.6.3
+    # via
+    #   -r requirements.in
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+markupsafe==1.1.1
+    # via
+    #   -r requirements.in
+    #   jinja2
+multimethod==1.3
+    # via -r requirements.in
+numpy==1.20.2
+    # via
+    #   -r requirements.in
+    #   ocrd-utils
+    #   opencv-python-headless
+ocrd-modelfactory==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-validators
+ocrd-models==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-validators
+ocrd-utils==2.24.0
+    # via
+    #   ocrd
+    #   ocrd-modelfactory
+    #   ocrd-models
+    #   ocrd-validators
+ocrd-validators==2.24.0
+    # via ocrd
+ocrd==2.24.0
+    # via -r requirements.in
+opencv-python-headless==4.5.1.48
+    # via ocrd
+pillow==8.2.0
+    # via ocrd-utils
+pyrsistent==0.17.3
+    # via jsonschema
+pyyaml==5.4.1
+    # via
+    #   ocrd
+    #   ocrd-validators
+requests==2.25.1
+    # via
+    #   bagit-profile
+    #   ocrd
+shapely==1.7.1
+    # via ocrd-validators
+six==1.15.0
+    # via jsonschema
+tqdm==4.60.0
+    # via -r requirements.in
+uniseg==0.7.1.post2
+    # via -r requirements.in
+urllib3==1.26.4
+    # via requests
+werkzeug==1.0.1
+    # via flask
+wrapt==1.12.1
+    # via deprecated
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements-dev.in

@@ -1,7 +0,0 @@
--c requirements.txt
-pip-tools
-pytest
-pytest-flake8
-pytest-cov
-pytest-mypy
-black

requirements-dev.txt

@@ -1,88 +1,5 @@
-#
+pytest
-# This file is autogenerated by pip-compile
+pytest-flake8
-# To update, run:
+pytest-cov
-#
+pytest-mypy
-#    pip-compile '.\requirements-dev.in'
+black
-#
-appdirs==1.4.4
-    # via black
-atomicwrites==1.4.0
-    # via
-    #   -c .\requirements.txt
-    #   pytest
-attrs==20.3.0
-    # via
-    #   -c .\requirements.txt
-    #   pytest
-    #   pytest-mypy
-black==21.5b0
-    # via -r .\requirements-dev.in
-click==7.1.2
-    # via
-    #   -c .\requirements.txt
-    #   black
-    #   pip-tools
-colorama==0.4.4
-    # via
-    #   -c .\requirements.txt
-    #   pytest
-coverage==5.5
-    # via pytest-cov
-filelock==3.0.12
-    # via pytest-mypy
-flake8==3.9.1
-    # via pytest-flake8
-iniconfig==1.1.1
-    # via pytest
-mccabe==0.6.1
-    # via flake8
-mypy-extensions==0.4.3
-    # via
-    #   black
-    #   mypy
-mypy==0.812
-    # via pytest-mypy
-packaging==20.9
-    # via pytest
-pathspec==0.8.1
-    # via black
-pep517==0.10.0
-    # via pip-tools
-pip-tools==6.1.0
-    # via -r .\requirements-dev.in
-pluggy==0.13.1
-    # via pytest
-py==1.10.0
-    # via pytest
-pycodestyle==2.7.0
-    # via flake8
-pyflakes==2.3.1
-    # via flake8
-pyparsing==2.4.7
-    # via packaging
-pytest-cov==2.11.1
-    # via -r .\requirements-dev.in
-pytest-flake8==1.0.7
-    # via -r .\requirements-dev.in
-pytest-mypy==0.8.1
-    # via -r .\requirements-dev.in
-pytest==6.2.4
-    # via
-    #   -r .\requirements-dev.in
-    #   pytest-cov
-    #   pytest-flake8
-    #   pytest-mypy
-regex==2021.4.4
-    # via black
-toml==0.10.2
-    # via
-    #   black
-    #   pep517
-    #   pytest
-typed-ast==1.4.3
-    # via mypy
-typing-extensions==3.10.0.0
-    # via mypy
-
-# The following packages are considered to be unsafe in a requirements file:
-# pip