From 35738eef8f049d36f6b9f645182ff6eefc56dc72 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 22 Nov 2021 02:07:12 +0100 Subject: [PATCH 01/23] add reusable workflows --- .github/workflows/BuildTheDocs.yml | 31 ++++++ .github/workflows/CoverageCollection.yml | 75 ++++++++++++++ .github/workflows/Package.yml | 48 +++++++++ .github/workflows/Params.yml | 42 ++++++++ .github/workflows/PublishOnPyPI.yml | 53 ++++++++++ .github/workflows/PublishToGitHubPages.yml | 62 +++++++++++ .github/workflows/Release.yml | 44 ++++++++ .github/workflows/StaticTypeCheck.yml | 53 ++++++++++ .github/workflows/UnitTesting.yml | 54 ++++++++++ .github/workflows/VerifyDocs.yml | 56 ++++++++++ ExamplePipeline.yml | 115 +++++++++++++++++++++ 11 files changed, 633 insertions(+) create mode 100644 .github/workflows/BuildTheDocs.yml create mode 100644 .github/workflows/CoverageCollection.yml create mode 100644 .github/workflows/Package.yml create mode 100644 .github/workflows/Params.yml create mode 100644 .github/workflows/PublishOnPyPI.yml create mode 100644 .github/workflows/PublishToGitHubPages.yml create mode 100644 .github/workflows/Release.yml create mode 100644 .github/workflows/StaticTypeCheck.yml create mode 100644 .github/workflows/UnitTesting.yml create mode 100644 .github/workflows/VerifyDocs.yml create mode 100644 ExamplePipeline.yml diff --git a/.github/workflows/BuildTheDocs.yml b/.github/workflows/BuildTheDocs.yml new file mode 100644 index 0000000..4870444 --- /dev/null +++ b/.github/workflows/BuildTheDocs.yml @@ -0,0 +1,31 @@ +name: Documentation + +on: + workflow_call: + inputs: + artifact: + description: 'Name of the documentation artifact.' + required: true + type: string + +jobs: + + BuildTheDocs: + name: ๐Ÿ““ Run BuildTheDocs + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ›ณ๏ธ Build documentation + uses: buildthedocs/btd@v0 + with: + skip-deploy: true + + - name: ๐Ÿ“ค Upload 'documentation' artifacts + uses: actions/upload-artifact@master + with: + name: ${{ inputs.artifact }} + path: doc/_build/html + retention-days: 7 diff --git a/.github/workflows/CoverageCollection.yml b/.github/workflows/CoverageCollection.yml new file mode 100644 index 0000000..32de7dd --- /dev/null +++ b/.github/workflows/CoverageCollection.yml @@ -0,0 +1,75 @@ +name: Coverage Collection + +on: + workflow_call: + inputs: + pyver: + description: 'Python version.' + required: false + default: '3.10' + type: string + artifact: + description: 'Name of the coverage artifact.' + required: true + type: string + secrets: + codacy_token: + description: 'Token to push result to codacy.' + required: true + +jobs: + + Coverage: + name: ๐Ÿ“ˆ Collect Coverage Data using Python ${{ inputs.pyver }} + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.pyver }} + + - name: ๐Ÿ—‚ Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/requirements.txt + + - name: Collect coverage + continue-on-error: true + run: | + python -m pytest -rA --cov=.. --cov-config=tests/.coveragerc tests/unit --color=yes + + - name: Convert to cobertura format + run: coverage xml + + - name: Convert to HTML format + run: | + coverage html + rm htmlcov/.gitignore + + - name: ๐Ÿ“ค Upload 'Coverage Report' artifact + continue-on-error: true + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: htmlcov + if-no-files-found: error + retention-days: 1 + + - name: ๐Ÿ“Š Publish coverage at CodeCov + continue-on-error: true + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml + flags: unittests + env_vars: PYTHON + + - name: ๐Ÿ“‰ Publish coverage at Codacy + continue-on-error: true + uses: codacy/codacy-coverage-reporter-action@master + with: + project-token: ${{ secrets.codacy_token }} + coverage-reports: ./coverage.xml diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml new file mode 100644 index 0000000..4d7e387 --- /dev/null +++ b/.github/workflows/Package.yml @@ -0,0 +1,48 @@ +name: Package + +on: + workflow_call: + inputs: + pyver: + description: 'Python version.' + required: false + default: '3.10' + type: string + artifact: + description: 'Name of the wheel artifact.' + required: true + type: string + +jobs: + + Package: + name: ๐Ÿ“ฆ Package in Wheel Format + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.pyver }} + + - name: ๐Ÿ”ง Install dependencies for packaging and release + run: | + python -m pip install --upgrade pip + pip install wheel + + - name: ๐Ÿ”จ Build Python package (source distribution) + run: python setup.py sdist + + - name: ๐Ÿ”จ Build Python package (binary distribution - wheel) + run: python setup.py bdist_wheel + + - name: ๐Ÿ“ค Upload wheel artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: dist/ + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml new file mode 100644 index 0000000..1352f81 --- /dev/null +++ b/.github/workflows/Params.yml @@ -0,0 +1,42 @@ +name: Params + +on: + workflow_call: + inputs: + pyver: + description: 'Python version.' + required: false + default: '3.10' + type: string + name: + description: 'Name of the tool.' + required: true + type: string + outputs: + params: + description: "Parameters to be used in other jobs." + value: ${{ jobs.Params.outputs.params }} + +jobs: + + Params: + runs-on: ubuntu-latest + outputs: + params: ${{ steps.params.outputs.params }} + steps: + + - id: params + shell: python + run: | + name = '${{ inputs.name }}' + params = { + 'package': name, + 'pyver': '${{ inputs.pyver }}', + 'artifacts': { + 'coverage': f'{name}-coverage', + 'typing': f'{name}-typing', + 'wheel': f'{name}-wheel', + 'doc': f'{name}-doc', + } + } + print(f'::set-output name=params::{params!s}') diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml new file mode 100644 index 0000000..5ab9235 --- /dev/null +++ b/.github/workflows/PublishOnPyPI.yml @@ -0,0 +1,53 @@ +name: Publish on PyPI + +on: + workflow_call: + inputs: + pyver: + description: 'Python version.' + required: false + default: '3.10' + type: string + artifact: + description: 'Name of the wheel artifact.' + required: true + type: string + secrets: + PYPI_TOKEN: + description: "Token for pushing releases to PyPI" + required: false + +jobs: + + PublishOnPyPI: + name: ๐Ÿš€ Publish to PyPI + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.artifact }}' from 'Package' job + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: dist/ + + - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.pyver }} + + - name: โš™ Install dependencies for packaging and release + run: | + python -m pip install --upgrade pip + pip install wheel twine + + - name: โคด Release Python package to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: | + twine upload dist/* + + - name: ๐Ÿ—‘๏ธ Delete packaging Artifacts + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.artifact }} diff --git a/.github/workflows/PublishToGitHubPages.yml b/.github/workflows/PublishToGitHubPages.yml new file mode 100644 index 0000000..64d2028 --- /dev/null +++ b/.github/workflows/PublishToGitHubPages.yml @@ -0,0 +1,62 @@ +name: Publish to GitHub Pages + +on: + workflow_call: + inputs: + doc: + description: 'Name of the documentation artifact.' + required: true + type: string + coverage: + description: 'Name of the coverage artifact.' + required: false + default: '' + type: string + typing: + description: 'Name of the typing artifact.' + required: false + default: '' + type: string + +jobs: + + PublishToGitHubPages: + name: ๐Ÿ“š Publish to GH-Pages + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.doc }}' from 'BuildTheDocs' job + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.doc }} + path: public + + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.coverage }}' from 'Coverage' job + if: ${{ inputs.coverage != '' }} + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.coverage }} + path: public/coverage + + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.typing }}' from 'StaticTypeCheck' job + if: ${{ inputs.typing != '' }} + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.typing }} + path: public/typing + + - name: '๐Ÿ““ Publish site to GitHub Pages' + if: github.event_name != 'pull_request' + run: | + cd public + touch .nojekyll + git init + cp ../.git/config ./.git/config + git add . + git config --local user.email "BuildTheDocs@GitHubActions" + git config --local user.name "GitHub Actions" + git commit -a -m "update ${{ github.sha }}" + git push -u origin +HEAD:gh-pages diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml new file mode 100644 index 0000000..561878e --- /dev/null +++ b/.github/workflows/Release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + workflow_call: + +jobs: + + Release: + name: ๐Ÿ“ Create 'Release Page' on GitHub + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ” Extract Git tag from GITHUB_REF + id: getVariables + run: | + GIT_TAG=${GITHUB_REF#refs/*/} + RELEASE_VERSION=${GIT_TAG#v} + RELEASE_DATETIME="$(date --utc '+%d.%m.%Y - %H:%M:%S')" + # write to step outputs + echo ::set-output name=gitTag::${GIT_TAG} + echo ::set-output name=version::${RELEASE_VERSION} + echo ::set-output name=datetime::${RELEASE_DATETIME} + + - name: ๐Ÿ“‘ Create Release Page + id: createReleasePage + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + tag_name: ${{ steps.getVariables.outputs.gitTag }} +# release_name: ${{ steps.getVariables.outputs.gitTag }} + body: | + **Automated Release created on: ${{ steps.getVariables.outputs.datetime }}** + + # New Features + * tbd + + # Changes + * tbd + + # Bug Fixes + * tbd + draft: false + prerelease: false diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml new file mode 100644 index 0000000..addac33 --- /dev/null +++ b/.github/workflows/StaticTypeCheck.yml @@ -0,0 +1,53 @@ +name: Static Type Check + +on: + workflow_call: + inputs: + package: + description: 'Name of the Python package.' + required: true + type: string + pyver: + description: 'Python version.' + required: false + default: '3.10' + type: string + artifact: + description: 'Name of the coverage artifact.' + required: true + type: string + +jobs: + + StaticTypeCheck: + name: ๐Ÿ‘€ Check Static Typing using Python ${{ inputs.pyver }} + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.pyver }} + + - name: ๐Ÿ—‚ Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/requirements.txt + + - name: Check Static Typing + continue-on-error: true + run: | + pwd + mypy --html-report htmlmypy -m ${{ inputs.package }} + + - name: ๐Ÿ“ค Upload 'Static Typing Report' artifact + continue-on-error: true + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: htmlmypy + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml new file mode 100644 index 0000000..72bd824 --- /dev/null +++ b/.github/workflows/UnitTesting.yml @@ -0,0 +1,54 @@ +name: Unit Testing + +on: + workflow_call: + inputs: + TestReport: + description: "Generate unit test report with junitxml and upload results as an artifact." + required: false + default: false + type: string + +jobs: + + UnitTesting: + name: ${{ matrix.icon }} Unit Tests using Python ${{ matrix.python }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: +# - {python: "3.6", icon: ๐Ÿ”ด} # until 23.12.2021 + - {python: "3.7", icon: ๐ŸŸ } # until 27.06.2023 + - {python: "3.8", icon: ๐ŸŸก} # until Oct. 2024 + - {python: "3.9", icon: ๐ŸŸข} # until Oct. 2025 + - {python: "3.10", icon: ๐ŸŸข} # until Oct. 2026 + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: ๐Ÿ”ง Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/requirements.txt + + - name: โ˜‘ Run unit tests + run: | + [ '${{ inputs.TestReport }}' = 'true' ] && PYTEST_ARGS='--junitxml=TestReport.xml' || unset PYTEST_ARGS + python -m pytest -rA tests/unit $PYTEST_ARGS --color=yes + + - name: ๐Ÿ“ค Upload 'TestReport.xml' artifact + if: inputs.TestReport == 'true' + uses: actions/upload-artifact@v2 + with: + name: TestReport-${{ matrix.python }} + path: TestReport.xml + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/VerifyDocs.yml b/.github/workflows/VerifyDocs.yml new file mode 100644 index 0000000..1e97e04 --- /dev/null +++ b/.github/workflows/VerifyDocs.yml @@ -0,0 +1,56 @@ +name: Verify examples + +on: + workflow_call: + inputs: + pyver: + description: 'Python version.' + required: false + default: '3.10' + type: string + +jobs: + + VerifyDocs: + name: ๐Ÿ‘ Verify example snippets using Python ${{ inputs.pyver }} + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.pyver }} + + - name: ๐Ÿ Install dependencies + run: | + pip3 install . + + - name: โœ‚ Extract code snippet from README + shell: python + run: | + from pathlib import Path + import re + + ROOT = Path('.') + + with (ROOT / 'README.md').open('r') as rptr: + content = rptr.read() + + m = re.search(r"```py(thon)?(?P.*?)```", content, re.MULTILINE|re.DOTALL) + + if m is None: + raise Exception("Regular expression did not find the example in the README!") + + with (ROOT / 'tests/docs/example.py').open('w') as wptr: + wptr.write(m["code"]) + + - name: Print example.py + run: cat tests/docs/example.py + + - name: โ˜‘ Run example snippet + working-directory: tests/docs + run: | + python3 example.py diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml new file mode 100644 index 0000000..971a373 --- /dev/null +++ b/ExamplePipeline.yml @@ -0,0 +1,115 @@ +name: Unit Testing, Coverage Collection, Package, Release, Documentation and Publish + +on: + workflow_dispatch: + +jobs: + + # This job is a workaround for global variables + # See https://github.com/actions/runner/issues/480 + Params: + uses: pyTooling/Actions/.github/workflows/Params.yml@dev + with: + name: ToolName + + UnitTesting: + uses: pyTooling/Actions/.github/workflows/UnitTesting.yml@dev + with: + TestReport: true + + Coverage: + uses: pyTooling/Actions/.github/workflows/CoverageCollection.yml@dev + needs: + - Params + with: + pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + secrets: + codacy_token: ${{ secrets.CODACY_PROJECT_TOKEN }} + + StaticTypeCheck: + uses: pyTooling/Actions/.github/workflows/StaticTypeCheck.yml@dev + needs: + - Params + with: + package: ${{ fromJson(needs.Params.outputs.params).package }} + pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + + Release: + uses: pyTooling/Actions/.github/workflows/Release.yml@dev + if: startsWith(github.ref, 'refs/tags') + needs: + - UnitTesting + - Coverage + - StaticTypeCheck + + Package: + uses: pyTooling/Actions/.github/workflows/Package.yml@dev + if: startsWith(github.ref, 'refs/tags') + needs: + - Params + - Coverage + with: + pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} + + PublishOnPyPI: + uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@dev + if: startsWith(github.ref, 'refs/tags') + needs: + - Params + - Release + - Package + with: + pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + + VerifyDocs: + uses: pyTooling/Actions/.github/workflows/VerifyDocs.yml@dev + needs: + - Params + with: + pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + + BuildTheDocs: + uses: pyTooling/Actions/.github/workflows/BuildTheDocs.yml@dev + needs: + - Params + - VerifyDocs + with: + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} + + PublishToGitHubPages: + uses: pyTooling/Actions/.github/workflows/PublishToGitHubPages.yml@dev + needs: + - Params + - BuildTheDocs + - Coverage + - StaticTypeCheck + with: + doc: ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} + coverage: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + typing: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + + + ArtifactCleanUp: + name: ๐Ÿ—‘๏ธ Artifact Cleanup + runs-on: ubuntu-latest + needs: + - Params + - Coverage + - StaticTypeCheck + - BuildTheDocs + - PublishToGitHubPages + + steps: + - name: ๐Ÿ—‘๏ธ Delete all Artifacts + uses: geekyeggo/delete-artifact@v1 + with: + name: | + ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} From 2a2aa3f0c842da791c83599cfc4cd0934d07e291 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:21:36 +0100 Subject: [PATCH 02/23] rename 'pyver' to 'python_version' --- .github/workflows/CoverageCollection.yml | 8 ++++---- .github/workflows/Package.yml | 6 +++--- .github/workflows/Params.yml | 4 ++-- .github/workflows/PublishOnPyPI.yml | 6 +++--- .github/workflows/StaticTypeCheck.yml | 8 ++++---- .github/workflows/VerifyDocs.yml | 6 +++--- ExamplePipeline.yml | 10 +++++----- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/CoverageCollection.yml b/.github/workflows/CoverageCollection.yml index 32de7dd..e1851a4 100644 --- a/.github/workflows/CoverageCollection.yml +++ b/.github/workflows/CoverageCollection.yml @@ -3,7 +3,7 @@ name: Coverage Collection on: workflow_call: inputs: - pyver: + python_version: description: 'Python version.' required: false default: '3.10' @@ -20,17 +20,17 @@ on: jobs: Coverage: - name: ๐Ÿ“ˆ Collect Coverage Data using Python ${{ inputs.pyver }} + name: ๐Ÿ“ˆ Collect Coverage Data using Python ${{ inputs.python_version }} runs-on: ubuntu-latest steps: - name: โฌ Checkout repository uses: actions/checkout@v2 - - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} uses: actions/setup-python@v2 with: - python-version: ${{ inputs.pyver }} + python-version: ${{ inputs.python_version }} - name: ๐Ÿ—‚ Install dependencies run: | diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index 4d7e387..6ff4eb9 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -3,7 +3,7 @@ name: Package on: workflow_call: inputs: - pyver: + python_version: description: 'Python version.' required: false default: '3.10' @@ -23,10 +23,10 @@ jobs: - name: ๐Ÿ“ฅ Checkout repository uses: actions/checkout@v2 - - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} uses: actions/setup-python@v2 with: - python-version: ${{ inputs.pyver }} + python-version: ${{ inputs.python_version }} - name: ๐Ÿ”ง Install dependencies for packaging and release run: | diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml index 1352f81..a87c387 100644 --- a/.github/workflows/Params.yml +++ b/.github/workflows/Params.yml @@ -3,7 +3,7 @@ name: Params on: workflow_call: inputs: - pyver: + python_version: description: 'Python version.' required: false default: '3.10' @@ -31,7 +31,7 @@ jobs: name = '${{ inputs.name }}' params = { 'package': name, - 'pyver': '${{ inputs.pyver }}', + 'python_version': '${{ inputs.python_version }}', 'artifacts': { 'coverage': f'{name}-coverage', 'typing': f'{name}-typing', diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml index 5ab9235..5f9dbac 100644 --- a/.github/workflows/PublishOnPyPI.yml +++ b/.github/workflows/PublishOnPyPI.yml @@ -3,7 +3,7 @@ name: Publish on PyPI on: workflow_call: inputs: - pyver: + python_version: description: 'Python version.' required: false default: '3.10' @@ -30,10 +30,10 @@ jobs: name: ${{ inputs.artifact }} path: dist/ - - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} uses: actions/setup-python@v2 with: - python-version: ${{ inputs.pyver }} + python-version: ${{ inputs.python_version }} - name: โš™ Install dependencies for packaging and release run: | diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml index addac33..170da67 100644 --- a/.github/workflows/StaticTypeCheck.yml +++ b/.github/workflows/StaticTypeCheck.yml @@ -7,7 +7,7 @@ on: description: 'Name of the Python package.' required: true type: string - pyver: + python_version: description: 'Python version.' required: false default: '3.10' @@ -20,17 +20,17 @@ on: jobs: StaticTypeCheck: - name: ๐Ÿ‘€ Check Static Typing using Python ${{ inputs.pyver }} + name: ๐Ÿ‘€ Check Static Typing using Python ${{ inputs.python_version }} runs-on: ubuntu-latest steps: - name: โฌ Checkout repository uses: actions/checkout@v2 - - name: ๐Ÿ Setup Python ${{ inputs.pyver }} + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} uses: actions/setup-python@v2 with: - python-version: ${{ inputs.pyver }} + python-version: ${{ inputs.python_version }} - name: ๐Ÿ—‚ Install dependencies run: | diff --git a/.github/workflows/VerifyDocs.yml b/.github/workflows/VerifyDocs.yml index 1e97e04..21ede91 100644 --- a/.github/workflows/VerifyDocs.yml +++ b/.github/workflows/VerifyDocs.yml @@ -3,7 +3,7 @@ name: Verify examples on: workflow_call: inputs: - pyver: + python_version: description: 'Python version.' required: false default: '3.10' @@ -12,7 +12,7 @@ on: jobs: VerifyDocs: - name: ๐Ÿ‘ Verify example snippets using Python ${{ inputs.pyver }} + name: ๐Ÿ‘ Verify example snippets using Python ${{ inputs.python_version }} runs-on: ubuntu-latest steps: @@ -22,7 +22,7 @@ jobs: - name: ๐Ÿ Setup Python uses: actions/setup-python@v2 with: - python-version: ${{ inputs.pyver }} + python-version: ${{ inputs.python_version }} - name: ๐Ÿ Install dependencies run: | diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 971a373..ace70b5 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -22,7 +22,7 @@ jobs: needs: - Params with: - pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} secrets: codacy_token: ${{ secrets.CODACY_PROJECT_TOKEN }} @@ -33,7 +33,7 @@ jobs: - Params with: package: ${{ fromJson(needs.Params.outputs.params).package }} - pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} Release: @@ -51,7 +51,7 @@ jobs: - Params - Coverage with: - pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} PublishOnPyPI: @@ -62,7 +62,7 @@ jobs: - Release - Package with: - pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} secrets: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} @@ -72,7 +72,7 @@ jobs: needs: - Params with: - pyver: ${{ fromJson(needs.Params.outputs.params).pyver }} + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} BuildTheDocs: uses: pyTooling/Actions/.github/workflows/BuildTheDocs.yml@dev From 7e559b5aeb1ac874889c3f18d55eeec4af8749fe Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:24:29 +0100 Subject: [PATCH 03/23] Package: add option 'requirements' --- .github/workflows/Package.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index 6ff4eb9..86447f6 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -8,6 +8,11 @@ on: required: false default: '3.10' type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: 'wheel' + type: string artifact: description: 'Name of the wheel artifact.' required: true @@ -30,8 +35,8 @@ jobs: - name: ๐Ÿ”ง Install dependencies for packaging and release run: | - python -m pip install --upgrade pip - pip install wheel + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} - name: ๐Ÿ”จ Build Python package (source distribution) run: python setup.py sdist From 39a93ee1d1b50aa79e2f2d5fb67ef408b8a0167a Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:26:55 +0100 Subject: [PATCH 04/23] PublishOnPyPI: add option 'requirements' --- .github/workflows/PublishOnPyPI.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml index 5f9dbac..948f0c5 100644 --- a/.github/workflows/PublishOnPyPI.yml +++ b/.github/workflows/PublishOnPyPI.yml @@ -8,6 +8,11 @@ on: required: false default: '3.10' type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: 'wheel twine' + type: string artifact: description: 'Name of the wheel artifact.' required: true @@ -37,15 +42,14 @@ jobs: - name: โš™ Install dependencies for packaging and release run: | - python -m pip install --upgrade pip - pip install wheel twine + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} - name: โคด Release Python package to PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - twine upload dist/* + run: twine upload dist/* - name: ๐Ÿ—‘๏ธ Delete packaging Artifacts uses: geekyeggo/delete-artifact@v1 From 40a8b3ff98cb50f7d556c8f827f6b419c691dce2 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:30:55 +0100 Subject: [PATCH 05/23] StaticTypeCheck: add option 'requirements' --- .github/workflows/StaticTypeCheck.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml index 170da67..6ec6d02 100644 --- a/.github/workflows/StaticTypeCheck.yml +++ b/.github/workflows/StaticTypeCheck.yml @@ -12,6 +12,11 @@ on: required: false default: '3.10' type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: '-r tests/requirements.txt' + type: string artifact: description: 'Name of the coverage artifact.' required: true @@ -34,14 +39,12 @@ jobs: - name: ๐Ÿ—‚ Install dependencies run: | - python -m pip install --upgrade pip - pip install -r tests/requirements.txt + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} - name: Check Static Typing continue-on-error: true - run: | - pwd - mypy --html-report htmlmypy -m ${{ inputs.package }} + run: mypy --html-report htmlmypy -m ${{ inputs.package }} - name: ๐Ÿ“ค Upload 'Static Typing Report' artifact continue-on-error: true From fe0ffaa72f91ff82d252d2ccc27daedd5b2659ed Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:32:08 +0100 Subject: [PATCH 06/23] UnitTesting: add option 'requirements' --- .github/workflows/UnitTesting.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml index 72bd824..04fef85 100644 --- a/.github/workflows/UnitTesting.yml +++ b/.github/workflows/UnitTesting.yml @@ -3,6 +3,11 @@ name: Unit Testing on: workflow_call: inputs: + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: '-r tests/requirements.txt' + type: string TestReport: description: "Generate unit test report with junitxml and upload results as an artifact." required: false @@ -36,8 +41,8 @@ jobs: - name: ๐Ÿ”ง Install dependencies run: | - python -m pip install --upgrade pip - pip install -r tests/requirements.txt + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} - name: โ˜‘ Run unit tests run: | From 498e7147259783044925eb48f565a44f79f84e27 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:33:24 +0100 Subject: [PATCH 07/23] CoverageCollection: add option 'requirements' --- .github/workflows/CoverageCollection.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CoverageCollection.yml b/.github/workflows/CoverageCollection.yml index e1851a4..ee4fa4d 100644 --- a/.github/workflows/CoverageCollection.yml +++ b/.github/workflows/CoverageCollection.yml @@ -8,6 +8,11 @@ on: required: false default: '3.10' type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: '-r tests/requirements.txt' + type: string artifact: description: 'Name of the coverage artifact.' required: true @@ -34,8 +39,8 @@ jobs: - name: ๐Ÿ—‚ Install dependencies run: | - python -m pip install --upgrade pip - pip install -r tests/requirements.txt + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} - name: Collect coverage continue-on-error: true From 93fd9c14c98d689f50e72264a494aa69f96850c9 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:35:08 +0100 Subject: [PATCH 08/23] PublishOnPyPI: do not delete artifact --- .github/workflows/PublishOnPyPI.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml index 948f0c5..650463c 100644 --- a/.github/workflows/PublishOnPyPI.yml +++ b/.github/workflows/PublishOnPyPI.yml @@ -50,8 +50,3 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: twine upload dist/* - - - name: ๐Ÿ—‘๏ธ Delete packaging Artifacts - uses: geekyeggo/delete-artifact@v1 - with: - name: ${{ inputs.artifact }} From 165cd5ec242df1f6a5a5d6ec99f0b3c49d4531f8 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:42:39 +0100 Subject: [PATCH 09/23] StaticTypeCheck: add options 'html_report' and 'mypy_args', remove 'package' --- .github/workflows/StaticTypeCheck.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml index 6ec6d02..5a97ebc 100644 --- a/.github/workflows/StaticTypeCheck.yml +++ b/.github/workflows/StaticTypeCheck.yml @@ -3,10 +3,6 @@ name: Static Type Check on: workflow_call: inputs: - package: - description: 'Name of the Python package.' - required: true - type: string python_version: description: 'Python version.' required: false @@ -17,6 +13,15 @@ on: required: false default: '-r tests/requirements.txt' type: string + html_report: + description: 'Directory for --html-report.' + required: false + default: 'htmlmypy' + type: string + mypy_args: + description: 'Arguments to mypy, except the HTML report (see option html_report).' + required: true + type: string artifact: description: 'Name of the coverage artifact.' required: true @@ -44,13 +49,16 @@ jobs: - name: Check Static Typing continue-on-error: true - run: mypy --html-report htmlmypy -m ${{ inputs.package }} + run: | + [ 'x${{ inputs.html_report }}' != 'x' ] && MYPY_HTML='--html-report=${{ inputs.html_report }}' || unset MYPY_HTML + mypy $MYPY_HTML ${{ inputs.mypy_args }} - name: ๐Ÿ“ค Upload 'Static Typing Report' artifact + if: ${{ inputs.html_report != '' }} continue-on-error: true uses: actions/upload-artifact@v2 with: name: ${{ inputs.artifact }} - path: htmlmypy + path: ${{ inputs.html_report }} if-no-files-found: error retention-days: 1 From fbaa2b582f65ed6133ac1218c961ddf84a6671c7 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 02:48:48 +0100 Subject: [PATCH 10/23] UnitTesting: generate matrix of jobs dynamically --- .github/workflows/UnitTesting.yml | 38 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml index 04fef85..cf7bdba 100644 --- a/.github/workflows/UnitTesting.yml +++ b/.github/workflows/UnitTesting.yml @@ -3,6 +3,11 @@ name: Unit Testing on: workflow_call: inputs: + python_versions: + description: 'Space separated list of Python versions to run tests with.' + required: false + default: '3.6 3.7 3.8 3.9 3.10' + type: string requirements: description: 'Python dependencies to be installed through pip.' required: false @@ -16,19 +21,40 @@ on: jobs: + + Versions: + runs-on: ubuntu-latest + outputs: + jobs: ${{ steps.versions.outputs.jobs }} + steps: + + - id: versions + shell: python + run: | + icons = { + '3.6': '๐Ÿ”ด', # until 23.12.2021 + '3.7': '๐ŸŸ ', # until 27.06.2023 + '3.8': '๐ŸŸก', # until Oct. 2024 + '3.9': '๐ŸŸข', # until Oct. 2025 + '3.10': '๐ŸŸข', # until Oct. 2026 + } + jobs = [ + {'python': version, 'icon': icons[version]} + for version in '${{ inputs.python_versions }}'.split(' ') + ] + print(f'::set-output name=jobs::{jobs!s}') + + UnitTesting: name: ${{ matrix.icon }} Unit Tests using Python ${{ matrix.python }} runs-on: ubuntu-latest + needs: + - Versions strategy: fail-fast: false matrix: - include: -# - {python: "3.6", icon: ๐Ÿ”ด} # until 23.12.2021 - - {python: "3.7", icon: ๐ŸŸ } # until 27.06.2023 - - {python: "3.8", icon: ๐ŸŸก} # until Oct. 2024 - - {python: "3.9", icon: ๐ŸŸข} # until Oct. 2025 - - {python: "3.10", icon: ๐ŸŸข} # until Oct. 2026 + include: ${{ fromJson(needs.Versions.outputs.jobs) }} steps: - name: โฌ Checkout repository From 7616643598dd78cc684aaa042b0c3e3d1b2aa182 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 03:16:06 +0100 Subject: [PATCH 11/23] generate matrix of jobs in workflow Params, instead of UnitTesting --- .github/workflows/Params.yml | 29 ++++++++++++++++++++++++++- .github/workflows/UnitTesting.yml | 33 +++---------------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml index a87c387..cd68544 100644 --- a/.github/workflows/Params.yml +++ b/.github/workflows/Params.yml @@ -8,6 +8,11 @@ on: required: false default: '3.10' type: string + python_version_list: + description: 'Space separated list of Python versions to run tests with.' + required: false + default: '3.6 3.7 3.8 3.9 3.10' + type: string name: description: 'Name of the tool.' required: true @@ -16,6 +21,9 @@ on: params: description: "Parameters to be used in other jobs." value: ${{ jobs.Params.outputs.params }} + python_jobs: + description: "List of Python versions to be used in the matrix of other jobs." + value: ${{ jobs.Params.outputs.python_jobs }} jobs: @@ -23,9 +31,11 @@ jobs: runs-on: ubuntu-latest outputs: params: ${{ steps.params.outputs.params }} + python_jobs: ${{ steps.params.outputs.python_jobs }} steps: - - id: params + - name: Generate 'params' and 'python_jobs' + id: params shell: python run: | name = '${{ inputs.name }}' @@ -40,3 +50,20 @@ jobs: } } print(f'::set-output name=params::{params!s}') + print("Params:") + print(params) + + data = { + '3.6': { 'icon': '๐Ÿ”ด', 'until': '23.12.2021' }, + '3.7': { 'icon': '๐ŸŸ ', 'until': '27.06.2023' }, + '3.8': { 'icon': '๐ŸŸก', 'until': 'Oct. 2024' }, + '3.9': { 'icon': '๐ŸŸข', 'until': 'Oct. 2025' }, + '3.10': { 'icon': '๐ŸŸข', 'until': 'Oct. 2026' }, + } + jobs = [ + {'python': version, 'icon': data[version]['icon']} + for version in '${{ inputs.python_version_list }}'.split(' ') + ] + print(f'::set-output name=python_jobs::{jobs!s}') + print("Python jobs:") + print(jobs) diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml index cf7bdba..45a83b0 100644 --- a/.github/workflows/UnitTesting.yml +++ b/.github/workflows/UnitTesting.yml @@ -3,10 +3,9 @@ name: Unit Testing on: workflow_call: inputs: - python_versions: + jobs: description: 'Space separated list of Python versions to run tests with.' - required: false - default: '3.6 3.7 3.8 3.9 3.10' + required: true type: string requirements: description: 'Python dependencies to be installed through pip.' @@ -21,40 +20,14 @@ on: jobs: - - Versions: - runs-on: ubuntu-latest - outputs: - jobs: ${{ steps.versions.outputs.jobs }} - steps: - - - id: versions - shell: python - run: | - icons = { - '3.6': '๐Ÿ”ด', # until 23.12.2021 - '3.7': '๐ŸŸ ', # until 27.06.2023 - '3.8': '๐ŸŸก', # until Oct. 2024 - '3.9': '๐ŸŸข', # until Oct. 2025 - '3.10': '๐ŸŸข', # until Oct. 2026 - } - jobs = [ - {'python': version, 'icon': icons[version]} - for version in '${{ inputs.python_versions }}'.split(' ') - ] - print(f'::set-output name=jobs::{jobs!s}') - - UnitTesting: name: ${{ matrix.icon }} Unit Tests using Python ${{ matrix.python }} runs-on: ubuntu-latest - needs: - - Versions strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.Versions.outputs.jobs) }} + include: ${{ fromJson(inputs.jobs) }} steps: - name: โฌ Checkout repository From dc22d814512dbf17c60813de0749277183217434 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 03:34:25 +0100 Subject: [PATCH 12/23] homogenize 'Checkout' icon --- .github/workflows/BuildTheDocs.yml | 2 +- .github/workflows/Package.yml | 2 +- .github/workflows/PublishToGitHubPages.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/BuildTheDocs.yml b/.github/workflows/BuildTheDocs.yml index 4870444..dbb3de7 100644 --- a/.github/workflows/BuildTheDocs.yml +++ b/.github/workflows/BuildTheDocs.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository + - name: โฌ Checkout repository uses: actions/checkout@v2 - name: ๐Ÿ›ณ๏ธ Build documentation diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index 86447f6..f637dfa 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: ๐Ÿ“ฅ Checkout repository + - name: โฌ Checkout repository uses: actions/checkout@v2 - name: ๐Ÿ Setup Python ${{ inputs.python_version }} diff --git a/.github/workflows/PublishToGitHubPages.yml b/.github/workflows/PublishToGitHubPages.yml index 64d2028..9a77da6 100644 --- a/.github/workflows/PublishToGitHubPages.yml +++ b/.github/workflows/PublishToGitHubPages.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout repository + - name: โฌ Checkout repository uses: actions/checkout@v2 - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.doc }}' from 'BuildTheDocs' job From 348b8d2258350f6c50baa8384af5d41ed1806284 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 03:42:58 +0100 Subject: [PATCH 13/23] update example pipeline --- ExamplePipeline.yml | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index ace70b5..c08ac2b 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -11,10 +11,16 @@ jobs: uses: pyTooling/Actions/.github/workflows/Params.yml@dev with: name: ToolName + # Optional + python_version: '3.10' + python_version_list: '3.8 3.9 3.10' UnitTesting: uses: pyTooling/Actions/.github/workflows/UnitTesting.yml@dev with: + jobs: ${{ needs.Params.outputs.python_jobs }} + # Optional + requirements: '-r tests/requirements.txt' TestReport: true Coverage: @@ -22,8 +28,10 @@ jobs: needs: - Params with: - python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: '-r tests/requirements.txt' secrets: codacy_token: ${{ secrets.CODACY_PROJECT_TOKEN }} @@ -32,9 +40,12 @@ jobs: needs: - Params with: - package: ${{ fromJson(needs.Params.outputs.params).package }} - python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + mypy_args: -m ${{ fromJson(needs.Params.outputs.params).package }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: '-r tests/requirements.txt' + html_report: 'htmlmypy' Release: uses: pyTooling/Actions/.github/workflows/Release.yml@dev @@ -51,8 +62,10 @@ jobs: - Params - Coverage with: - python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: 'wheel' PublishOnPyPI: uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@dev @@ -62,8 +75,10 @@ jobs: - Release - Package with: - python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: 'wheel twine' secrets: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} @@ -72,6 +87,7 @@ jobs: needs: - Params with: + # Optional python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} BuildTheDocs: @@ -91,6 +107,7 @@ jobs: - StaticTypeCheck with: doc: ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} + # Optional coverage: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} typing: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} From e7a0ac0b7a2b45c57dc1c93261ed04ec4c9ecd0b Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 22:36:23 +0100 Subject: [PATCH 14/23] rename 'params.package' to 'params.name' and 'params.artifacts.wheel' to 'params.artifacts.package' --- .github/workflows/Package.yml | 4 ++-- .github/workflows/Params.yml | 4 ++-- .github/workflows/PublishOnPyPI.yml | 2 +- ExamplePipeline.yml | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml index f637dfa..ebcc572 100644 --- a/.github/workflows/Package.yml +++ b/.github/workflows/Package.yml @@ -14,14 +14,14 @@ on: default: 'wheel' type: string artifact: - description: 'Name of the wheel artifact.' + description: 'Name of the package artifact.' required: true type: string jobs: Package: - name: ๐Ÿ“ฆ Package in Wheel Format + name: ๐Ÿ“ฆ Package in Source and Wheel Format runs-on: ubuntu-latest steps: diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml index cd68544..5d6fc5e 100644 --- a/.github/workflows/Params.yml +++ b/.github/workflows/Params.yml @@ -40,12 +40,12 @@ jobs: run: | name = '${{ inputs.name }}' params = { - 'package': name, + 'name': name, 'python_version': '${{ inputs.python_version }}', 'artifacts': { 'coverage': f'{name}-coverage', 'typing': f'{name}-typing', - 'wheel': f'{name}-wheel', + 'package': f'{name}-package', 'doc': f'{name}-doc', } } diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml index 650463c..eb4c702 100644 --- a/.github/workflows/PublishOnPyPI.yml +++ b/.github/workflows/PublishOnPyPI.yml @@ -14,7 +14,7 @@ on: default: 'wheel twine' type: string artifact: - description: 'Name of the wheel artifact.' + description: 'Name of the package artifact.' required: true type: string secrets: diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index c08ac2b..1f76246 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -40,7 +40,7 @@ jobs: needs: - Params with: - mypy_args: -m ${{ fromJson(needs.Params.outputs.params).package }} + mypy_args: -m ${{ fromJson(needs.Params.outputs.params).name }} artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} # Optional python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} @@ -62,7 +62,7 @@ jobs: - Params - Coverage with: - artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.package }} # Optional python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} requirements: 'wheel' @@ -75,7 +75,7 @@ jobs: - Release - Package with: - artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.wheel }} + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.package }} # Optional python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} requirements: 'wheel twine' From dd3bd9752afe0b715a6e188e3fc1db8a8ac1fa60 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 22:38:39 +0100 Subject: [PATCH 15/23] BuildTheDocs: set retention-days to 1 --- .github/workflows/BuildTheDocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/BuildTheDocs.yml b/.github/workflows/BuildTheDocs.yml index dbb3de7..28fbe18 100644 --- a/.github/workflows/BuildTheDocs.yml +++ b/.github/workflows/BuildTheDocs.yml @@ -28,4 +28,4 @@ jobs: with: name: ${{ inputs.artifact }} path: doc/_build/html - retention-days: 7 + retention-days: 1 From b141c8c0ef1370a40c9d9c22ea52c0ea0d1af21f Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 22:41:54 +0100 Subject: [PATCH 16/23] PublishOnPyPI: publish source and wheel in two steps --- .github/workflows/PublishOnPyPI.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml index eb4c702..04cca8f 100644 --- a/.github/workflows/PublishOnPyPI.yml +++ b/.github/workflows/PublishOnPyPI.yml @@ -45,8 +45,14 @@ jobs: python -m pip install -U pip python -m pip install ${{ inputs.requirements }} - - name: โคด Release Python package to PyPI + - name: โคด Release Python source package to PyPI env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: twine upload dist/* + run: twine upload dist/*.tar.gz + + - name: โคด Release Python wheel package to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/*.whl From c63201364662a0ddde87b4cb936465b1c38f939f Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 22:42:45 +0100 Subject: [PATCH 17/23] StaticTypeCheck: fix description of option 'artifact' --- .github/workflows/StaticTypeCheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml index 5a97ebc..1855542 100644 --- a/.github/workflows/StaticTypeCheck.yml +++ b/.github/workflows/StaticTypeCheck.yml @@ -23,7 +23,7 @@ on: required: true type: string artifact: - description: 'Name of the coverage artifact.' + description: 'Name of the typing artifact.' required: true type: string From 6e501b3ead0390801e030c7d5ecced5383d4686a Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 22:49:56 +0100 Subject: [PATCH 18/23] UnitTesting: make the artifact name an option; generate it in Params --- .github/workflows/Params.yml | 1 + .github/workflows/UnitTesting.yml | 8 ++++---- ExamplePipeline.yml | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml index 5d6fc5e..6817022 100644 --- a/.github/workflows/Params.yml +++ b/.github/workflows/Params.yml @@ -43,6 +43,7 @@ jobs: 'name': name, 'python_version': '${{ inputs.python_version }}', 'artifacts': { + 'unittesting': f'{name}-TestReport', 'coverage': f'{name}-coverage', 'typing': f'{name}-typing', 'package': f'{name}-package', diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml index 45a83b0..47c53b8 100644 --- a/.github/workflows/UnitTesting.yml +++ b/.github/workflows/UnitTesting.yml @@ -12,10 +12,10 @@ on: required: false default: '-r tests/requirements.txt' type: string - TestReport: + artifact: description: "Generate unit test report with junitxml and upload results as an artifact." required: false - default: false + default: '' type: string jobs: @@ -45,14 +45,14 @@ jobs: - name: โ˜‘ Run unit tests run: | - [ '${{ inputs.TestReport }}' = 'true' ] && PYTEST_ARGS='--junitxml=TestReport.xml' || unset PYTEST_ARGS + [ 'x${{ inputs.artifact }}' != 'x' ] && PYTEST_ARGS='--junitxml=TestReport.xml' || unset PYTEST_ARGS python -m pytest -rA tests/unit $PYTEST_ARGS --color=yes - name: ๐Ÿ“ค Upload 'TestReport.xml' artifact if: inputs.TestReport == 'true' uses: actions/upload-artifact@v2 with: - name: TestReport-${{ matrix.python }} + name: ${{ inputs.artifact }}-${{ matrix.python }} path: TestReport.xml if-no-files-found: error retention-days: 1 diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 1f76246..4e69223 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -21,7 +21,7 @@ jobs: jobs: ${{ needs.Params.outputs.python_jobs }} # Optional requirements: '-r tests/requirements.txt' - TestReport: true + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }} Coverage: uses: pyTooling/Actions/.github/workflows/CoverageCollection.yml@dev From a6077abb3d42902d0a9740482c2a5994c70e3eb3 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 22:51:28 +0100 Subject: [PATCH 19/23] ExamplePipeline: update ArtifactCleanUp --- ExamplePipeline.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 4e69223..23ece50 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -123,10 +123,19 @@ jobs: - PublishToGitHubPages steps: - - name: ๐Ÿ—‘๏ธ Delete all Artifacts + + - name: ๐Ÿ—‘๏ธ Delete package Artifacts + if: ${{ ! startsWith(github.ref, 'refs/tags') }} uses: geekyeggo/delete-artifact@v1 with: name: | + ${{ fromJson(needs.Params.outputs.params).artifacts.package }} + + - name: ๐Ÿ—‘๏ธ Delete remaining Artifacts + uses: geekyeggo/delete-artifact@v1 + with: + name: | + ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }}-* ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} From 72278fee1d9cfb095a37d3c82aec66cbc767fe41 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 23:13:12 +0100 Subject: [PATCH 20/23] PublishOnPyPI: delete artifact --- .github/workflows/PublishOnPyPI.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml index 04cca8f..44dd83a 100644 --- a/.github/workflows/PublishOnPyPI.yml +++ b/.github/workflows/PublishOnPyPI.yml @@ -56,3 +56,8 @@ jobs: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: twine upload dist/*.whl + + - name: ๐Ÿ—‘๏ธ Delete packaging Artifacts + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.artifact }} From 462e51253c95686d1b8fbbe64a69f521fb36fe25 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 29 Nov 2021 23:40:10 +0100 Subject: [PATCH 21/23] StaticTypeCheck: require the user to specify the mypy commands --- .github/workflows/Params.yml | 1 - .github/workflows/StaticTypeCheck.yml | 16 +++++++--------- ExamplePipeline.yml | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml index 6817022..2f74ef2 100644 --- a/.github/workflows/Params.yml +++ b/.github/workflows/Params.yml @@ -40,7 +40,6 @@ jobs: run: | name = '${{ inputs.name }}' params = { - 'name': name, 'python_version': '${{ inputs.python_version }}', 'artifacts': { 'unittesting': f'{name}-TestReport', diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml index 1855542..2facb91 100644 --- a/.github/workflows/StaticTypeCheck.yml +++ b/.github/workflows/StaticTypeCheck.yml @@ -13,13 +13,13 @@ on: required: false default: '-r tests/requirements.txt' type: string - html_report: - description: 'Directory for --html-report.' + report: + description: 'Directory to upload as an artifact.' required: false default: 'htmlmypy' type: string - mypy_args: - description: 'Arguments to mypy, except the HTML report (see option html_report).' + commands: + description: 'Commands to run the static type checks.' required: true type: string artifact: @@ -49,16 +49,14 @@ jobs: - name: Check Static Typing continue-on-error: true - run: | - [ 'x${{ inputs.html_report }}' != 'x' ] && MYPY_HTML='--html-report=${{ inputs.html_report }}' || unset MYPY_HTML - mypy $MYPY_HTML ${{ inputs.mypy_args }} + run: ${{ inputs.commands }} - name: ๐Ÿ“ค Upload 'Static Typing Report' artifact - if: ${{ inputs.html_report != '' }} + if: ${{ inputs.artifact != '' }} continue-on-error: true uses: actions/upload-artifact@v2 with: name: ${{ inputs.artifact }} - path: ${{ inputs.html_report }} + path: ${{ inputs.report }} if-no-files-found: error retention-days: 1 diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 23ece50..3368b15 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -40,12 +40,12 @@ jobs: needs: - Params with: - mypy_args: -m ${{ fromJson(needs.Params.outputs.params).name }} + commands: mypy --html-report htmlmypy -p ToolName artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} # Optional python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} requirements: '-r tests/requirements.txt' - html_report: 'htmlmypy' + report: 'htmlmypy' Release: uses: pyTooling/Actions/.github/workflows/Release.yml@dev From 8e3ff6387cffeacdf4b0653f4fe96f355a4c770b Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 30 Nov 2021 00:25:39 +0100 Subject: [PATCH 22/23] ExamplePipeline: change '@dev' to '@main' --- ExamplePipeline.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 3368b15..9d7a655 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -8,7 +8,7 @@ jobs: # This job is a workaround for global variables # See https://github.com/actions/runner/issues/480 Params: - uses: pyTooling/Actions/.github/workflows/Params.yml@dev + uses: pyTooling/Actions/.github/workflows/Params.yml@main with: name: ToolName # Optional @@ -16,7 +16,7 @@ jobs: python_version_list: '3.8 3.9 3.10' UnitTesting: - uses: pyTooling/Actions/.github/workflows/UnitTesting.yml@dev + uses: pyTooling/Actions/.github/workflows/UnitTesting.yml@main with: jobs: ${{ needs.Params.outputs.python_jobs }} # Optional @@ -24,7 +24,7 @@ jobs: artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }} Coverage: - uses: pyTooling/Actions/.github/workflows/CoverageCollection.yml@dev + uses: pyTooling/Actions/.github/workflows/CoverageCollection.yml@main needs: - Params with: @@ -36,7 +36,7 @@ jobs: codacy_token: ${{ secrets.CODACY_PROJECT_TOKEN }} StaticTypeCheck: - uses: pyTooling/Actions/.github/workflows/StaticTypeCheck.yml@dev + uses: pyTooling/Actions/.github/workflows/StaticTypeCheck.yml@main needs: - Params with: @@ -48,7 +48,7 @@ jobs: report: 'htmlmypy' Release: - uses: pyTooling/Actions/.github/workflows/Release.yml@dev + uses: pyTooling/Actions/.github/workflows/Release.yml@main if: startsWith(github.ref, 'refs/tags') needs: - UnitTesting @@ -56,7 +56,7 @@ jobs: - StaticTypeCheck Package: - uses: pyTooling/Actions/.github/workflows/Package.yml@dev + uses: pyTooling/Actions/.github/workflows/Package.yml@main if: startsWith(github.ref, 'refs/tags') needs: - Params @@ -68,7 +68,7 @@ jobs: requirements: 'wheel' PublishOnPyPI: - uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@dev + uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@main if: startsWith(github.ref, 'refs/tags') needs: - Params @@ -83,7 +83,7 @@ jobs: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} VerifyDocs: - uses: pyTooling/Actions/.github/workflows/VerifyDocs.yml@dev + uses: pyTooling/Actions/.github/workflows/VerifyDocs.yml@main needs: - Params with: @@ -91,7 +91,7 @@ jobs: python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} BuildTheDocs: - uses: pyTooling/Actions/.github/workflows/BuildTheDocs.yml@dev + uses: pyTooling/Actions/.github/workflows/BuildTheDocs.yml@main needs: - Params - VerifyDocs @@ -99,7 +99,7 @@ jobs: artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} PublishToGitHubPages: - uses: pyTooling/Actions/.github/workflows/PublishToGitHubPages.yml@dev + uses: pyTooling/Actions/.github/workflows/PublishToGitHubPages.yml@main needs: - Params - BuildTheDocs From 7115cf79be8dca989a1c0b94e644586a3be1be06 Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 30 Nov 2021 00:27:30 +0100 Subject: [PATCH 23/23] add reusable workflow 'ArtifactCleanUp' --- .github/workflows/ArtifactCleanUp.yml | 34 +++++++++++++++++++++++++++ ExamplePipeline.yml | 29 +++++++---------------- 2 files changed, 42 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/ArtifactCleanUp.yml diff --git a/.github/workflows/ArtifactCleanUp.yml b/.github/workflows/ArtifactCleanUp.yml new file mode 100644 index 0000000..ed77abf --- /dev/null +++ b/.github/workflows/ArtifactCleanUp.yml @@ -0,0 +1,34 @@ +name: ArtifactCleanUp + +on: + workflow_call: + inputs: + package: + description: 'Artifacts to be removed on not tagged runs.' + required: true + type: string + remaining: + description: 'Artifacts to be removed unconditionally.' + required: false + default: '' + type: string + +jobs: + + ArtifactCleanUp: + name: ๐Ÿ—‘๏ธ Artifact Cleanup + runs-on: ubuntu-latest + + steps: + + - name: ๐Ÿ—‘๏ธ Delete package Artifacts + if: ${{ ! startsWith(github.ref, 'refs/tags') }} + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.package }} + + - name: ๐Ÿ—‘๏ธ Delete remaining Artifacts + if: ${{ inputs.remaining != '' }} + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.remaining }} diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 9d7a655..b0b14fa 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -111,31 +111,18 @@ jobs: coverage: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} typing: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} - ArtifactCleanUp: - name: ๐Ÿ—‘๏ธ Artifact Cleanup - runs-on: ubuntu-latest + uses: pyTooling/Actions/.github/workflows/ArtifactCleanUp.yml@main needs: - Params - Coverage - StaticTypeCheck - BuildTheDocs - PublishToGitHubPages - - steps: - - - name: ๐Ÿ—‘๏ธ Delete package Artifacts - if: ${{ ! startsWith(github.ref, 'refs/tags') }} - uses: geekyeggo/delete-artifact@v1 - with: - name: | - ${{ fromJson(needs.Params.outputs.params).artifacts.package }} - - - name: ๐Ÿ—‘๏ธ Delete remaining Artifacts - uses: geekyeggo/delete-artifact@v1 - with: - name: | - ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }}-* - ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} - ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} - ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} + with: + package: ${{ fromJson(needs.Params.outputs.params).artifacts.package }} + remaining: | + ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }}-* + ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + ${{ fromJson(needs.Params.outputs.params).artifacts.doc }}