diff --git a/.github/workflows/CoverageCollection.yml b/.github/workflows/CoverageCollection.yml index ff95257..f5bde28 100644 --- a/.github/workflows/CoverageCollection.yml +++ b/.github/workflows/CoverageCollection.yml @@ -35,6 +35,16 @@ on: required: false default: '-r tests/requirements.txt' type: string + unittest_directory: + description: 'Path to the directory containing unit tests.' + required: false + default: 'tests/unit' + type: string + coverage_config: + description: 'Path to the .coveragerc file. Use pyproject.toml by default.' + required: false + default: 'pyproject.toml' + type: string artifact: description: 'Name of the coverage artifact.' required: true @@ -62,27 +72,68 @@ jobs: - name: 🗂 Install dependencies run: | python -m pip install -U pip + python -m pip install tomli python -m pip install ${{ inputs.requirements }} + - name: 🔁 Extract configurations from pyproject.toml + id: getVariables + shell: python + run: | + from pathlib import Path + from tomli import load as tomli_load + + htmlDirectory = 'htmlcov' + xmlFile = './coverage.xml' + coverageRC = "${{ inputs.coverage_config }}".strip() + + # Read output paths from 'pyproject.toml' file + if coverageRC == "pyproject.toml": + pyProjectFile = Path("pyproject.toml") + if pyProjectFile.exists(): + with pyProjectFile.open("rb") as file: + pyProjectSettings = tomli_load(file) + + htmlDirectory = pyProjectSettings["tool"]["coverage"]["html"]["directory"] + xmlFile = pyProjectSettings["tool"]["coverage"]["xml"]["output"] + else: + print(f"File '{pyProjectFile}' not found and no ' .coveragerc' file specified.") + + # Read output paths from '.coveragerc' file + elif len(coverageRC) > 0: + coverageRCFile = Path(coverageRC) + if coverageRCFile.exists(): + with coverageRCFile.open("rb") as file: + coverageRCSettings = tomli_load(file) + + htmlDirectory = coverageRCSettings["html"]["directory"] + xmlFile = coverageRCSettings["xml"]["output"] + else: + print(f"File '{coverageRCFile}' not found.") + + print(f"::set-output name=coverage_report_html_directory::{htmlDirectory}") + print(f"::set-output name=coverage_report_xml::{xmlFile}") + print(f"DEBUG:\n html={htmlDirectory}\n xml={xmlFile}") + - name: Collect coverage continue-on-error: true run: | - python -m pytest -rA --cov=.. --cov-config=tests/.coveragerc tests/unit --color=yes + [ 'x${{ inputs.coverage_config }}' != 'x' ] && PYCOV_ARGS='--cov-config=${{ inputs.coverage_config }}' || unset PYCOV_ARGS + python -m pytest -rA --cov=. $PYCOV_ARGS ${{ inputs.unittest_directory }} --color=yes - name: Convert to cobertura format run: coverage xml - name: Convert to HTML format run: | - coverage html - rm htmlcov/.gitignore + coverage html -d ${{ steps.getVariables.outputs.coverage_report_html_directory }} + rm ${{ steps.getVariables.outputs.coverage_report_html_directory }}/.gitignore - name: 📤 Upload 'Coverage Report' artifact continue-on-error: true uses: actions/upload-artifact@v2 with: name: ${{ inputs.artifact }} - path: htmlcov + path: ${{ steps.getVariables.outputs.coverage_report_html_directory }} if-no-files-found: error retention-days: 1 @@ -90,7 +141,7 @@ jobs: continue-on-error: true uses: codecov/codecov-action@v1 with: - file: ./coverage.xml + file: ${{ steps.getVariables.outputs.coverage_report_xml }} flags: unittests env_vars: PYTHON @@ -99,4 +150,4 @@ jobs: uses: codacy/codacy-coverage-reporter-action@master with: project-token: ${{ secrets.codacy_token }} - coverage-reports: ./coverage.xml + coverage-reports: ${{ steps.getVariables.outputs.coverage_report_xml }} diff --git a/.github/workflows/Parameters.yml b/.github/workflows/Parameters.yml index f7b217d..085dc13 100644 --- a/.github/workflows/Parameters.yml +++ b/.github/workflows/Parameters.yml @@ -75,16 +75,19 @@ jobs: print("Parameters:") print(params) + versions = '${{ inputs.python_version_list }}'.split(' ') + if '3.6' in versions: + print('WARNING: support for Python 3.6 ended in 2021.12.23') data = { - '3.6': { 'icon': '🔴', 'until': '23.12.2021' }, - '3.7': { 'icon': '🟠', 'until': '27.06.2023' }, - '3.8': { 'icon': '🟡', 'until': 'Oct. 2024' }, - '3.9': { 'icon': '🟢', 'until': 'Oct. 2025' }, - '3.10': { 'icon': '🟢', 'until': 'Oct. 2026' }, + '3.6': { 'icon': '⚫', 'until': '2021.12.23' }, + '3.7': { 'icon': '🔴', 'until': '2023.06.27' }, + '3.8': { 'icon': '🟠', 'until': '2024.10' }, + '3.9': { 'icon': '🟡', 'until': '2025.10' }, + '3.10': { 'icon': '🟢', 'until': '2026.10' }, } jobs = [ {'python': version, 'icon': data[version]['icon']} - for version in '${{ inputs.python_version_list }}'.split(' ') + for version in versions ] print(f'::set-output name=python_jobs::{jobs!s}') print("Python jobs:") diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml index 4155898..ada82b4 100644 --- a/.github/workflows/UnitTesting.yml +++ b/.github/workflows/UnitTesting.yml @@ -34,6 +34,11 @@ on: required: false default: '-r tests/requirements.txt' type: string + unittest_directory: + description: 'Path to the directory containing unit tests.' + required: false + default: 'tests/unit' + type: string artifact: description: "Generate unit test report with junitxml and upload results as an artifact." required: false @@ -68,7 +73,7 @@ jobs: - name: ☑ Run unit tests run: | [ 'x${{ inputs.artifact }}' != 'x' ] && PYTEST_ARGS='--junitxml=TestReport.xml' || unset PYTEST_ARGS - python -m pytest -rA tests/unit $PYTEST_ARGS --color=yes + python -m pytest -rA ${{ inputs.unittest_directory }} $PYTEST_ARGS --color=yes - name: 📤 Upload 'TestReport.xml' artifact if: inputs.artifact != '' diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml index 9be2d54..8e4af52 100644 --- a/ExamplePipeline.yml +++ b/ExamplePipeline.yml @@ -148,7 +148,7 @@ jobs: uses: pyTooling/Actions/.github/workflows/ArtifactCleanUp.yml@main needs: - Params - - UnitTesting + - PublishTestResults - Coverage - StaticTypeCheck - BuildTheDocs diff --git a/README.md b/README.md index d57a14b..495acfc 100644 --- a/README.md +++ b/README.md @@ -7,24 +7,30 @@ language for writing reusable CI code. However, Python being equally popular and capable, usage of JS/TS might be bypassed, with some caveats. This repository gathers reusable CI tooling for testing, packaging and distributing Python projects and documentation. + ## Context -GitHub Actions supports four types of reusable code: +GitHub Actions supports five procedures to reuse code: -- JavaScript Action. +- JavaScript Action: - [docs.github.com: actions/creating-actions/creating-a-javascript-action](https://docs.github.com/en/actions/creating-actions/creating-a-javascript-action) -- Container Action. +- Container Action: - [docs.github.com: actions/creating-actions/creating-a-docker-container-action](https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action) -- Composite Action. +- Container Step: + - [docs.github.com: actions/learn-github-actions/workflow-syntax-for-github-actions#example-using-a-docker-public-registry-action](https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-using-a-docker-public-registry-action) + - [docs.github.com: actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idstepswithargs](https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#jobsjob_idstepswithargs) +- Composite Action: - [docs.github.com: actions/creating-actions/creating-a-composite-action](https://docs.github.com/en/actions/creating-actions/creating-a-composite-action) - [github.blog/changelog: 2020-08-07-github-actions-composite-run-steps](https://github.blog/changelog/2020-08-07-github-actions-composite-run-steps/) - [github.blog/changelog: 2021-08-25-github-actions-reduce-duplication-with-action-compositio](https://github.blog/changelog/2021-08-25-github-actions-reduce-duplication-with-action-composition/) -- Reusable Workflows. +- Reusable Workflow: - [docs.github.com: actions/learn-github-actions/reusing-workflows](https://docs.github.com/en/actions/learn-github-actions/reusing-workflows) - [github.blog/changelog: 2021-10-05-github-actions-dry-your-github-actions-configuration-by-reusing-workflows](https://github.blog/changelog/2021-10-05-github-actions-dry-your-github-actions-configuration-by-reusing-workflows/) -Leaving JavaScript and Container Actions aside, the main differences between Composite Actions and Reusable Workflows -are the following: +Container Actions and Container Steps are almost equivalent: Actions use a configuration file (`action.yml`), while +Steps do not. +Leaving JavaScript and Container Actions and Steps aside, the main differences between Composite Actions and Reusable +Workflows are the following: - Composite Actions can be executed from a remote/external path or from the checked out branch, and from any location. However, Reusable Workflows can only be used through a remote/external path (`{owner}/{repo}/{path}/{filename}@{ref}`), @@ -74,6 +80,7 @@ It allows using the `post` feature with scripts written in bash, python or any o the environment. See: [actions/runner#1478](https://github.com/actions/runner/issues/1478). + ## Reusable workflows This repository provides 10+ Reusable Workflows based on the CI pipelines of the repos in this organisation, @@ -88,13 +95,16 @@ As shown in the screenshot above, the expected order is: - Global: - [Parameters](.github/workflows/Parameters.yml): a workaround for the limitations to handle global variables in GitHub Actions workflows (see [actions/runner#480](https://github.com/actions/runner/issues/480)). - It generates outputs with artifact names and job matrices to be used in other jobs. + It generates outputs with artifact names and job matrices to be used in later running jobs. - Code testing/analysis: - [UnitTesting](.github/workflows/UnitTesting.yml): run unit test with `pytest` using multiple versions of Python, and - optionally upload results as XML reports. - - [CoverageCollection](.github/workflows/CoverageCollection.yml): collect coverage data with `pytest` using a single - version of Python, generate HTML and Cobertura (XML) reports, upload the HTML report as an artifact, and upload the - results to Codecov and Codacy. + optionally upload results as XML reports. Configuration options to `pytest` should be given via section + `[tool.pytest.ini_options]` in a `pyproject.toml` file. + - [CoverageCollection](.github/workflows/CoverageCollection.yml): collect code coverage data (incl. branch coverage) + with `pytest`/`pytest-cov`/`coverage.py` using a single version of Python (latest). It generates HTML and Cobertura + (XML)reports, upload the HTML report as an artifact, and upload the test results to Codecov and Codacy. Configuration + options to `pytest` and `coverage.py` should be given via section `[tool.pytest.ini_options]` and `[tool.coverage.*]` + in a `pyproject.toml` file. - [StaticTypeCheck](.github/workflows/StaticTypeCheck.yml): collect static type check result with `mypy`, and optionally upload results as an HTML report. Example `commands`: @@ -121,7 +131,7 @@ As shown in the screenshot above, the expected order is: mypy --html-report ../htmlmypy -p ToolName ``` - - [VerifyDocs](.github/workflows/VerifyDocs.yml): extract code examples from the README and test. + - [VerifyDocs](.github/workflows/VerifyDocs.yml): extract code examples from the README and test these code snippets. - Packaging and releasing: - [Release](.github/workflows/Release.yml): publish GitHub Release. - [Package](.github/workflows/Package.yml): generate source and wheel packages, and upload them as an artifact. @@ -150,19 +160,23 @@ Find further usage cases in the following list of projects: - [VHDL/pyVHDLModel](https://github.com/VHDL/pyVHDLModel/tree/main/.github/workflows) +## References + +- [hdl/containers#48](https://github.com/hdl/containers/issues/48) + + ## Contributors * [Patrick Lehmann](https://GitHub.com/Paebbels) * [Unai Martinez-Corral](https://GitHub.com/umarcor) (Maintainer) -* [and more...](https://GitHub.com/pyTooling/Actions/graphs/contributors) +* [and more...](https://GitHub.com/pyTooling/Actions/graphs/contributors) ## License -This Python package (source code) licensed under [Apache License 2.0](LICENSE.md). +This Python package (source code) licensed under [Apache License 2.0](LICENSE.md). The accompanying documentation is licensed under [Creative Commons - Attribution 4.0 (CC-BY 4.0)](doc/Doc-License.rst). +--- -------------------------- - -SPDX-License-Identifier: Apache-2.0 \ No newline at end of file +SPDX-License-Identifier: Apache-2.0 diff --git a/releaser/DEVELOPMENT.md b/releaser/DEVELOPMENT.md new file mode 100644 index 0000000..3fdc386 --- /dev/null +++ b/releaser/DEVELOPMENT.md @@ -0,0 +1,8 @@ +# Releaser Development + +- [pyTooling/pyAttributes](https://github.com/pyTooling/pyAttributes) or + [willmcgugan/rich](https://github.com/willmcgugan/rich) might be used to enhance the UX. + +- It might be desirable to have pyTooling.Version.SemVersion handle the regular expression from + [semver.org](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string), and use + proper Python classes in **Releaser**. diff --git a/releaser/README.md b/releaser/README.md index 4949b53..16cf0a2 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -48,6 +48,20 @@ as assets. In this context, one of the main use cases of **Releaser** is pushing artifacts as release assets. Thus, the name of the Action. +GitHub provides an official CLI tool, written in golang: [cli/cli](https://github.com/cli/cli). +When the Python version of **Releaser** was written, `cli` was evaluated as an alternative to *PyGitHub*. +`gh release` was (and still is) not flexible enough to update the reference of a release, without deleting and +recreating it (see [cli.github.com: manual/gh_release_create](https://cli.github.com/manual/gh_release_create)). +Deletion and recreation is unfortunate, because it notifies all the watchers of a repository +(see [eine/tip#111](https://github.com/eine/tip/issues/111)). +However, [cli.github.com: manual/gh_release_upload](https://cli.github.com/manual/gh_release_upload) handles uploading +artifacts as assets faster and with better stability for larger files than *PyGitHub* +(see [msys2/msys2-installer#36](https://github.com/msys2/msys2-installer/pull/36)). +Furthermore, the GitHub CLI is installed on GitHub Actions' default virtual environments. +Although `gh` does not support login through SSH (see [cli/cli#3715](https://github.com/cli/cli/issues/3715)), on GitHub +Actions a token is available `${{ github.token }}`. +Therefore, **Releaser** uses `gh release upload` internally. + ## Usage The following block shows a minimal YAML workflow file: