From 15e584f0ab9b57025b65f96594c2b6264e3ee8b6 Mon Sep 17 00:00:00 2001 From: Benjamin Rosemann Date: Wed, 5 May 2021 15:20:35 +0200 Subject: [PATCH] Introduce version pinning and license checcking Introduces version pinning via pip-tools for reproducible builds. Introduces license checking via pip-licenses and CircleCI. Licenses are only checked for new builds and when requirements.txt changes. The list of allowed licenses is kept in a separate file to be able to distinguish changes in CI-Configuration/Tools and license list changes. --- .allowed-licenses | 34 ++++++++++++ .circleci/config.yml | 43 ++++++++++++++- requirements-dev.in | 7 +++ requirements-dev.txt | 93 +++++++++++++++++++++++++++++-- requirements.in | 11 ++++ requirements.txt | 127 +++++++++++++++++++++++++++++++++++++++---- 6 files changed, 296 insertions(+), 19 deletions(-) create mode 100644 .allowed-licenses create mode 100644 requirements-dev.in create mode 100644 requirements.in diff --git a/.allowed-licenses b/.allowed-licenses new file mode 100644 index 0000000..995a54f --- /dev/null +++ b/.allowed-licenses @@ -0,0 +1,34 @@ +# Allowed licenses for this project, one license per line + +# Variants of the Apache Software License +Apache +Apache License 2.0 +Apache Software License +Apache Software License, BSD License + +# Variants of the BSD License +BSD +BSD License + +# Variants of the LGPL +GNU Library or Lesser General Public License (LGPL) +LGPL + +# Variants of the HPND License +Historical Permission Notice and Disclaimer (HPND) + +# Variants of the MIT License +MIT +MIT License +MIT License, Mozilla Public License 2.0 (MPL 2.0) + +# Variants of the MLP +MPL +MPL 2.0 +Mozilla Public License 2.0 (MPL 2.0) + +# Public Domain Variants +Public Domain + +# Variants of the Python Software Foundation License +Python Software Foundation License diff --git a/.circleci/config.yml b/.circleci/config.yml index 1033fb0..553acd7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,10 +9,41 @@ jobs: - image: circleci/python:<< parameters.python-version >> steps: - checkout - - run: pip3 install --upgrade pip - - run: pip3 install -r requirements.txt - - run: pip3 install pytest + - &install + run: + name: Install packages + command: | + pip3 install --upgrade pip + pip3 install -r requirements.txt + - run: + name: Install specific packages + command: pip3 install pytest - run: pytest + license-scan: + docker: + - image: circleci/python:3.9 + steps: + - checkout + - run: # Only run license checks if we have changes in requirements.txt + name: Check for changes + command: | + if [ -z "<< pipeline.git.base_revision >>" ]; then + echo "No previous build, run license check by default." + elif git diff-tree --no-commit-id --name-only -r << pipeline.git.revision >> << pipeline.git.base_revision >> | grep requirements.txt ; then + echo "Changes in requirements.txt, run license check." + else + echo "No relevant changes found, skip running license check." + circleci-agent step halt + fi + - *install + - run: + name: Install specific packages + command: pip3 install pip-licenses + - run: # Read allowed licenses from file to ";"-separated string while removing empty lines and comments + name: License check + command: | + ALLOWED=$(sed -e '/^#.*$/d' -e '/^\s*$/d' .allowed-licenses | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/;/g'); echo $ALLOWED + pip-licenses --from=mixed --allow-only="${ALLOWED}" --summary workflows: all-tests: @@ -21,3 +52,9 @@ workflows: matrix: parameters: python-version: ["3.5", "3.6", "3.7", "3.8", "3.9"] + license-scan: + jobs: + - license-scan: + filters: + branches: + only: master diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..b4fdf84 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,7 @@ +-c requirements.txt +pip-tools +pytest +pytest-flake8 +pytest-cov +pytest-mypy +black diff --git a/requirements-dev.txt b/requirements-dev.txt index 9403f15..81a0a77 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,88 @@ -pytest -pytest-flake8 -pytest-cov -pytest-mypy -black +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile '.\requirements-dev.in' +# +appdirs==1.4.4 + # via black +atomicwrites==1.4.0 + # via + # -c .\requirements.txt + # pytest +attrs==20.3.0 + # via + # -c .\requirements.txt + # pytest + # pytest-mypy +black==21.5b0 + # via -r .\requirements-dev.in +click==7.1.2 + # via + # -c .\requirements.txt + # black + # pip-tools +colorama==0.4.4 + # via + # -c .\requirements.txt + # pytest +coverage==5.5 + # via pytest-cov +filelock==3.0.12 + # via pytest-mypy +flake8==3.9.1 + # via pytest-flake8 +iniconfig==1.1.1 + # via pytest +mccabe==0.6.1 + # via flake8 +mypy-extensions==0.4.3 + # via + # black + # mypy +mypy==0.812 + # via pytest-mypy +packaging==20.9 + # via pytest +pathspec==0.8.1 + # via black +pep517==0.10.0 + # via pip-tools +pip-tools==6.1.0 + # via -r .\requirements-dev.in +pluggy==0.13.1 + # via pytest +py==1.10.0 + # via pytest +pycodestyle==2.7.0 + # via flake8 +pyflakes==2.3.1 + # via flake8 +pyparsing==2.4.7 + # via packaging +pytest-cov==2.11.1 + # via -r .\requirements-dev.in +pytest-flake8==1.0.7 + # via -r .\requirements-dev.in +pytest-mypy==0.8.1 + # via -r .\requirements-dev.in +pytest==6.2.4 + # via + # -r .\requirements-dev.in + # pytest-cov + # pytest-flake8 + # pytest-mypy +regex==2021.4.4 + # via black +toml==0.10.2 + # via + # black + # pep517 + # pytest +typed-ast==1.4.3 + # via mypy +typing-extensions==3.10.0.0 + # via mypy + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..7bb53ac --- /dev/null +++ b/requirements.in @@ -0,0 +1,11 @@ +click +jinja2 +lxml +uniseg +numpy +colorama +MarkupSafe +ocrd >= 2.20.1 +attrs +multimethod == 1.3 # latest version to officially support Python 3.5 +tqdm diff --git a/requirements.txt b/requirements.txt index 7bb53ac..32a5f8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,116 @@ -click -jinja2 -lxml -uniseg -numpy -colorama -MarkupSafe -ocrd >= 2.20.1 -attrs -multimethod == 1.3 # latest version to officially support Python 3.5 -tqdm +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile '.\requirements.in' +# +atomicwrites==1.4.0 + # via ocrd-utils +attrs==20.3.0 + # via + # -r .\requirements.in + # jsonschema +bagit-profile==1.3.1 + # via + # ocrd + # ocrd-validators +bagit==1.8.1 + # via + # bagit-profile + # ocrd + # ocrd-validators +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +click==7.1.2 + # via + # -r .\requirements.in + # flask + # ocrd + # ocrd-validators +colorama==0.4.4 + # via -r .\requirements.in +deprecated==1.2.0 + # via ocrd +flask==1.1.2 + # via ocrd +idna==2.10 + # via requests +itsdangerous==1.1.0 + # via flask +jinja2==2.11.3 + # via + # -r .\requirements.in + # flask +jsonschema==3.2.0 + # via + # ocrd + # ocrd-validators +lxml==4.6.3 + # via + # -r .\requirements.in + # ocrd + # ocrd-modelfactory + # ocrd-models +markupsafe==1.1.1 + # via + # -r .\requirements.in + # jinja2 +multimethod==1.3 + # via -r .\requirements.in +numpy==1.20.2 + # via + # -r .\requirements.in + # ocrd-utils + # opencv-python-headless +ocrd-modelfactory==2.24.0 + # via + # ocrd + # ocrd-validators +ocrd-models==2.24.0 + # via + # ocrd + # ocrd-modelfactory + # ocrd-validators +ocrd-utils==2.24.0 + # via + # ocrd + # ocrd-modelfactory + # ocrd-models + # ocrd-validators +ocrd-validators==2.24.0 + # via ocrd +ocrd==2.24.0 + # via -r .\requirements.in +opencv-python-headless==4.5.1.48 + # via ocrd +pillow==8.2.0 + # via ocrd-utils +pyrsistent==0.17.3 + # via jsonschema +pyyaml==5.4.1 + # via + # ocrd + # ocrd-validators +requests==2.25.1 + # via + # bagit-profile + # ocrd +shapely==1.7.1 + # via ocrd-validators +six==1.15.0 + # via jsonschema +tqdm==4.60.0 + # via -r .\requirements.in +uniseg==0.7.1.post2 + # via -r .\requirements.in +urllib3==1.26.4 + # via requests +werkzeug==1.0.1 + # via flask +wrapt==1.12.1 + # via deprecated + +# The following packages are considered to be unsafe in a requirements file: +# setuptools