diff --git a/.github/workflows/ArtifactCleanUp.yml b/.github/workflows/ArtifactCleanUp.yml new file mode 100644 index 0000000..ed77abf --- /dev/null +++ b/.github/workflows/ArtifactCleanUp.yml @@ -0,0 +1,34 @@ +name: ArtifactCleanUp + +on: + workflow_call: + inputs: + package: + description: 'Artifacts to be removed on not tagged runs.' + required: true + type: string + remaining: + description: 'Artifacts to be removed unconditionally.' + required: false + default: '' + type: string + +jobs: + + ArtifactCleanUp: + name: ๐Ÿ—‘๏ธ Artifact Cleanup + runs-on: ubuntu-latest + + steps: + + - name: ๐Ÿ—‘๏ธ Delete package Artifacts + if: ${{ ! startsWith(github.ref, 'refs/tags') }} + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.package }} + + - name: ๐Ÿ—‘๏ธ Delete remaining Artifacts + if: ${{ inputs.remaining != '' }} + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.remaining }} diff --git a/.github/workflows/BuildTheDocs.yml b/.github/workflows/BuildTheDocs.yml new file mode 100644 index 0000000..28fbe18 --- /dev/null +++ b/.github/workflows/BuildTheDocs.yml @@ -0,0 +1,31 @@ +name: Documentation + +on: + workflow_call: + inputs: + artifact: + description: 'Name of the documentation artifact.' + required: true + type: string + +jobs: + + BuildTheDocs: + name: ๐Ÿ““ Run BuildTheDocs + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ›ณ๏ธ Build documentation + uses: buildthedocs/btd@v0 + with: + skip-deploy: true + + - name: ๐Ÿ“ค Upload 'documentation' artifacts + uses: actions/upload-artifact@master + with: + name: ${{ inputs.artifact }} + path: doc/_build/html + retention-days: 1 diff --git a/.github/workflows/CoverageCollection.yml b/.github/workflows/CoverageCollection.yml new file mode 100644 index 0000000..ee4fa4d --- /dev/null +++ b/.github/workflows/CoverageCollection.yml @@ -0,0 +1,80 @@ +name: Coverage Collection + +on: + workflow_call: + inputs: + python_version: + description: 'Python version.' + required: false + default: '3.10' + type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: '-r tests/requirements.txt' + type: string + artifact: + description: 'Name of the coverage artifact.' + required: true + type: string + secrets: + codacy_token: + description: 'Token to push result to codacy.' + required: true + +jobs: + + Coverage: + name: ๐Ÿ“ˆ Collect Coverage Data using Python ${{ inputs.python_version }} + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python_version }} + + - name: ๐Ÿ—‚ Install dependencies + run: | + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} + + - name: Collect coverage + continue-on-error: true + run: | + python -m pytest -rA --cov=.. --cov-config=tests/.coveragerc tests/unit --color=yes + + - name: Convert to cobertura format + run: coverage xml + + - name: Convert to HTML format + run: | + coverage html + rm htmlcov/.gitignore + + - name: ๐Ÿ“ค Upload 'Coverage Report' artifact + continue-on-error: true + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: htmlcov + if-no-files-found: error + retention-days: 1 + + - name: ๐Ÿ“Š Publish coverage at CodeCov + continue-on-error: true + uses: codecov/codecov-action@v1 + with: + file: ./coverage.xml + flags: unittests + env_vars: PYTHON + + - name: ๐Ÿ“‰ Publish coverage at Codacy + continue-on-error: true + uses: codacy/codacy-coverage-reporter-action@master + with: + project-token: ${{ secrets.codacy_token }} + coverage-reports: ./coverage.xml diff --git a/.github/workflows/Package.yml b/.github/workflows/Package.yml new file mode 100644 index 0000000..ebcc572 --- /dev/null +++ b/.github/workflows/Package.yml @@ -0,0 +1,53 @@ +name: Package + +on: + workflow_call: + inputs: + python_version: + description: 'Python version.' + required: false + default: '3.10' + type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: 'wheel' + type: string + artifact: + description: 'Name of the package artifact.' + required: true + type: string + +jobs: + + Package: + name: ๐Ÿ“ฆ Package in Source and Wheel Format + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python_version }} + + - name: ๐Ÿ”ง Install dependencies for packaging and release + run: | + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} + + - name: ๐Ÿ”จ Build Python package (source distribution) + run: python setup.py sdist + + - name: ๐Ÿ”จ Build Python package (binary distribution - wheel) + run: python setup.py bdist_wheel + + - name: ๐Ÿ“ค Upload wheel artifact + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: dist/ + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/Params.yml b/.github/workflows/Params.yml new file mode 100644 index 0000000..2f74ef2 --- /dev/null +++ b/.github/workflows/Params.yml @@ -0,0 +1,69 @@ +name: Params + +on: + workflow_call: + inputs: + python_version: + description: 'Python version.' + required: false + default: '3.10' + type: string + python_version_list: + description: 'Space separated list of Python versions to run tests with.' + required: false + default: '3.6 3.7 3.8 3.9 3.10' + type: string + name: + description: 'Name of the tool.' + required: true + type: string + outputs: + params: + description: "Parameters to be used in other jobs." + value: ${{ jobs.Params.outputs.params }} + python_jobs: + description: "List of Python versions to be used in the matrix of other jobs." + value: ${{ jobs.Params.outputs.python_jobs }} + +jobs: + + Params: + runs-on: ubuntu-latest + outputs: + params: ${{ steps.params.outputs.params }} + python_jobs: ${{ steps.params.outputs.python_jobs }} + steps: + + - name: Generate 'params' and 'python_jobs' + id: params + shell: python + run: | + name = '${{ inputs.name }}' + params = { + 'python_version': '${{ inputs.python_version }}', + 'artifacts': { + 'unittesting': f'{name}-TestReport', + 'coverage': f'{name}-coverage', + 'typing': f'{name}-typing', + 'package': f'{name}-package', + 'doc': f'{name}-doc', + } + } + print(f'::set-output name=params::{params!s}') + print("Params:") + print(params) + + data = { + '3.6': { 'icon': '๐Ÿ”ด', 'until': '23.12.2021' }, + '3.7': { 'icon': '๐ŸŸ ', 'until': '27.06.2023' }, + '3.8': { 'icon': '๐ŸŸก', 'until': 'Oct. 2024' }, + '3.9': { 'icon': '๐ŸŸข', 'until': 'Oct. 2025' }, + '3.10': { 'icon': '๐ŸŸข', 'until': 'Oct. 2026' }, + } + jobs = [ + {'python': version, 'icon': data[version]['icon']} + for version in '${{ inputs.python_version_list }}'.split(' ') + ] + print(f'::set-output name=python_jobs::{jobs!s}') + print("Python jobs:") + print(jobs) diff --git a/.github/workflows/PublishOnPyPI.yml b/.github/workflows/PublishOnPyPI.yml new file mode 100644 index 0000000..44dd83a --- /dev/null +++ b/.github/workflows/PublishOnPyPI.yml @@ -0,0 +1,63 @@ +name: Publish on PyPI + +on: + workflow_call: + inputs: + python_version: + description: 'Python version.' + required: false + default: '3.10' + type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: 'wheel twine' + type: string + artifact: + description: 'Name of the package artifact.' + required: true + type: string + secrets: + PYPI_TOKEN: + description: "Token for pushing releases to PyPI" + required: false + +jobs: + + PublishOnPyPI: + name: ๐Ÿš€ Publish to PyPI + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.artifact }}' from 'Package' job + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: dist/ + + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python_version }} + + - name: โš™ Install dependencies for packaging and release + run: | + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} + + - name: โคด Release Python source package to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/*.tar.gz + + - name: โคด Release Python wheel package to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + run: twine upload dist/*.whl + + - name: ๐Ÿ—‘๏ธ Delete packaging Artifacts + uses: geekyeggo/delete-artifact@v1 + with: + name: ${{ inputs.artifact }} diff --git a/.github/workflows/PublishToGitHubPages.yml b/.github/workflows/PublishToGitHubPages.yml new file mode 100644 index 0000000..9a77da6 --- /dev/null +++ b/.github/workflows/PublishToGitHubPages.yml @@ -0,0 +1,62 @@ +name: Publish to GitHub Pages + +on: + workflow_call: + inputs: + doc: + description: 'Name of the documentation artifact.' + required: true + type: string + coverage: + description: 'Name of the coverage artifact.' + required: false + default: '' + type: string + typing: + description: 'Name of the typing artifact.' + required: false + default: '' + type: string + +jobs: + + PublishToGitHubPages: + name: ๐Ÿ“š Publish to GH-Pages + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.doc }}' from 'BuildTheDocs' job + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.doc }} + path: public + + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.coverage }}' from 'Coverage' job + if: ${{ inputs.coverage != '' }} + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.coverage }} + path: public/coverage + + - name: ๐Ÿ“ฅ Download artifacts '${{ inputs.typing }}' from 'StaticTypeCheck' job + if: ${{ inputs.typing != '' }} + uses: actions/download-artifact@v2 + with: + name: ${{ inputs.typing }} + path: public/typing + + - name: '๐Ÿ““ Publish site to GitHub Pages' + if: github.event_name != 'pull_request' + run: | + cd public + touch .nojekyll + git init + cp ../.git/config ./.git/config + git add . + git config --local user.email "BuildTheDocs@GitHubActions" + git config --local user.name "GitHub Actions" + git commit -a -m "update ${{ github.sha }}" + git push -u origin +HEAD:gh-pages diff --git a/.github/workflows/Release.yml b/.github/workflows/Release.yml new file mode 100644 index 0000000..561878e --- /dev/null +++ b/.github/workflows/Release.yml @@ -0,0 +1,44 @@ +name: Release + +on: + workflow_call: + +jobs: + + Release: + name: ๐Ÿ“ Create 'Release Page' on GitHub + runs-on: ubuntu-latest + + steps: + - name: ๐Ÿ” Extract Git tag from GITHUB_REF + id: getVariables + run: | + GIT_TAG=${GITHUB_REF#refs/*/} + RELEASE_VERSION=${GIT_TAG#v} + RELEASE_DATETIME="$(date --utc '+%d.%m.%Y - %H:%M:%S')" + # write to step outputs + echo ::set-output name=gitTag::${GIT_TAG} + echo ::set-output name=version::${RELEASE_VERSION} + echo ::set-output name=datetime::${RELEASE_DATETIME} + + - name: ๐Ÿ“‘ Create Release Page + id: createReleasePage + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + tag_name: ${{ steps.getVariables.outputs.gitTag }} +# release_name: ${{ steps.getVariables.outputs.gitTag }} + body: | + **Automated Release created on: ${{ steps.getVariables.outputs.datetime }}** + + # New Features + * tbd + + # Changes + * tbd + + # Bug Fixes + * tbd + draft: false + prerelease: false diff --git a/.github/workflows/StaticTypeCheck.yml b/.github/workflows/StaticTypeCheck.yml new file mode 100644 index 0000000..2facb91 --- /dev/null +++ b/.github/workflows/StaticTypeCheck.yml @@ -0,0 +1,62 @@ +name: Static Type Check + +on: + workflow_call: + inputs: + python_version: + description: 'Python version.' + required: false + default: '3.10' + type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: '-r tests/requirements.txt' + type: string + report: + description: 'Directory to upload as an artifact.' + required: false + default: 'htmlmypy' + type: string + commands: + description: 'Commands to run the static type checks.' + required: true + type: string + artifact: + description: 'Name of the typing artifact.' + required: true + type: string + +jobs: + + StaticTypeCheck: + name: ๐Ÿ‘€ Check Static Typing using Python ${{ inputs.python_version }} + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ inputs.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python_version }} + + - name: ๐Ÿ—‚ Install dependencies + run: | + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} + + - name: Check Static Typing + continue-on-error: true + run: ${{ inputs.commands }} + + - name: ๐Ÿ“ค Upload 'Static Typing Report' artifact + if: ${{ inputs.artifact != '' }} + continue-on-error: true + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }} + path: ${{ inputs.report }} + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/UnitTesting.yml b/.github/workflows/UnitTesting.yml new file mode 100644 index 0000000..47c53b8 --- /dev/null +++ b/.github/workflows/UnitTesting.yml @@ -0,0 +1,58 @@ +name: Unit Testing + +on: + workflow_call: + inputs: + jobs: + description: 'Space separated list of Python versions to run tests with.' + required: true + type: string + requirements: + description: 'Python dependencies to be installed through pip.' + required: false + default: '-r tests/requirements.txt' + type: string + artifact: + description: "Generate unit test report with junitxml and upload results as an artifact." + required: false + default: '' + type: string + +jobs: + + UnitTesting: + name: ${{ matrix.icon }} Unit Tests using Python ${{ matrix.python }} + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(inputs.jobs) }} + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + + - name: ๐Ÿ”ง Install dependencies + run: | + python -m pip install -U pip + python -m pip install ${{ inputs.requirements }} + + - name: โ˜‘ Run unit tests + run: | + [ 'x${{ inputs.artifact }}' != 'x' ] && PYTEST_ARGS='--junitxml=TestReport.xml' || unset PYTEST_ARGS + python -m pytest -rA tests/unit $PYTEST_ARGS --color=yes + + - name: ๐Ÿ“ค Upload 'TestReport.xml' artifact + if: inputs.TestReport == 'true' + uses: actions/upload-artifact@v2 + with: + name: ${{ inputs.artifact }}-${{ matrix.python }} + path: TestReport.xml + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/VerifyDocs.yml b/.github/workflows/VerifyDocs.yml new file mode 100644 index 0000000..21ede91 --- /dev/null +++ b/.github/workflows/VerifyDocs.yml @@ -0,0 +1,56 @@ +name: Verify examples + +on: + workflow_call: + inputs: + python_version: + description: 'Python version.' + required: false + default: '3.10' + type: string + +jobs: + + VerifyDocs: + name: ๐Ÿ‘ Verify example snippets using Python ${{ inputs.python_version }} + runs-on: ubuntu-latest + + steps: + - name: โฌ Checkout repository + uses: actions/checkout@v2 + + - name: ๐Ÿ Setup Python + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python_version }} + + - name: ๐Ÿ Install dependencies + run: | + pip3 install . + + - name: โœ‚ Extract code snippet from README + shell: python + run: | + from pathlib import Path + import re + + ROOT = Path('.') + + with (ROOT / 'README.md').open('r') as rptr: + content = rptr.read() + + m = re.search(r"```py(thon)?(?P.*?)```", content, re.MULTILINE|re.DOTALL) + + if m is None: + raise Exception("Regular expression did not find the example in the README!") + + with (ROOT / 'tests/docs/example.py').open('w') as wptr: + wptr.write(m["code"]) + + - name: Print example.py + run: cat tests/docs/example.py + + - name: โ˜‘ Run example snippet + working-directory: tests/docs + run: | + python3 example.py diff --git a/ExamplePipeline.png b/ExamplePipeline.png new file mode 100644 index 0000000..9e4779c Binary files /dev/null and b/ExamplePipeline.png differ diff --git a/ExamplePipeline.yml b/ExamplePipeline.yml new file mode 100644 index 0000000..40ac7a9 --- /dev/null +++ b/ExamplePipeline.yml @@ -0,0 +1,130 @@ +name: Unit Testing, Coverage Collection, Package, Release, Documentation and Publish + +on: + workflow_dispatch: + +jobs: + + # This job is a workaround for global variables + # See https://github.com/actions/runner/issues/480 + Params: + uses: pyTooling/Actions/.github/workflows/Params.yml@main + with: + name: ToolName + # Optional + python_version: '3.10' + python_version_list: '3.8 3.9 3.10' + + UnitTesting: + uses: pyTooling/Actions/.github/workflows/UnitTesting.yml@main + needs: + - Params + with: + jobs: ${{ needs.Params.outputs.python_jobs }} + # Optional + requirements: '-r tests/requirements.txt' + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }} + + Coverage: + uses: pyTooling/Actions/.github/workflows/CoverageCollection.yml@main + needs: + - Params + with: + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: '-r tests/requirements.txt' + secrets: + codacy_token: ${{ secrets.CODACY_PROJECT_TOKEN }} + + StaticTypeCheck: + uses: pyTooling/Actions/.github/workflows/StaticTypeCheck.yml@main + needs: + - Params + with: + commands: mypy --html-report htmlmypy -p ToolName + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: '-r tests/requirements.txt' + report: 'htmlmypy' + + Release: + uses: pyTooling/Actions/.github/workflows/Release.yml@main + if: startsWith(github.ref, 'refs/tags') + needs: + - UnitTesting + - Coverage + - StaticTypeCheck + + Package: + uses: pyTooling/Actions/.github/workflows/Package.yml@main + if: startsWith(github.ref, 'refs/tags') + needs: + - Params + - Coverage + with: + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.package }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: 'wheel' + + PublishOnPyPI: + uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@main + if: startsWith(github.ref, 'refs/tags') + needs: + - Params + - Release + - Package + with: + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.package }} + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + requirements: 'wheel twine' + secrets: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + + VerifyDocs: + uses: pyTooling/Actions/.github/workflows/VerifyDocs.yml@main + needs: + - Params + with: + # Optional + python_version: ${{ fromJson(needs.Params.outputs.params).python_version }} + + BuildTheDocs: + uses: pyTooling/Actions/.github/workflows/BuildTheDocs.yml@main + needs: + - Params + - VerifyDocs + with: + artifact: ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} + + PublishToGitHubPages: + uses: pyTooling/Actions/.github/workflows/PublishToGitHubPages.yml@main + needs: + - Params + - BuildTheDocs + - Coverage + - StaticTypeCheck + with: + doc: ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} + # Optional + coverage: ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + typing: ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + + ArtifactCleanUp: + uses: pyTooling/Actions/.github/workflows/ArtifactCleanUp.yml@main + needs: + - Params + - Coverage + - StaticTypeCheck + - BuildTheDocs + - PublishToGitHubPages + with: + package: ${{ fromJson(needs.Params.outputs.params).artifacts.package }} + remaining: | + ${{ fromJson(needs.Params.outputs.params).artifacts.unittesting }}-* + ${{ fromJson(needs.Params.outputs.params).artifacts.coverage }} + ${{ fromJson(needs.Params.outputs.params).artifacts.typing }} + ${{ fromJson(needs.Params.outputs.params).artifacts.doc }} diff --git a/README.md b/README.md index 3c74755..de87c38 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,149 @@ # Actions Reusable steps and workflows for GitHub Actions, focused on Python packages. + +GitHub Actions workflows, actions and documentation are mostly focused on JavaScript/TypeScript as the scripting +language for writing reusable CI code. +However, Python being equally popular and capable, usage of JS/TS might be bypassed, with some caveats. +This repository gathers reusable CI tooling for testing, packaging and distributing Python projects and documentation. + +## Context + +GitHub Actions supports four types of reusable code: + +- JavaScript Action. + - [docs.github.com: actions/creating-actions/creating-a-javascript-action](https://docs.github.com/en/actions/creating-actions/creating-a-javascript-action) +- Container Action. + - [docs.github.com: actions/creating-actions/creating-a-docker-container-action](https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action) +- Composite Action. + - [docs.github.com: actions/creating-actions/creating-a-composite-action](https://docs.github.com/en/actions/creating-actions/creating-a-composite-action) + - [github.blog/changelog: 2020-08-07-github-actions-composite-run-steps](https://github.blog/changelog/2020-08-07-github-actions-composite-run-steps/) + - [github.blog/changelog: 2021-08-25-github-actions-reduce-duplication-with-action-compositio](https://github.blog/changelog/2021-08-25-github-actions-reduce-duplication-with-action-composition/) +- Reusable Workflows. + - [docs.github.com: actions/learn-github-actions/reusing-workflows](https://docs.github.com/en/actions/learn-github-actions/reusing-workflows) + - [github.blog/changelog: 2021-10-05-github-actions-dry-your-github-actions-configuration-by-reusing-workflows](https://github.blog/changelog/2021-10-05-github-actions-dry-your-github-actions-configuration-by-reusing-workflows/) + +Leaving JavaScript and Container Actions aside, the main differences between Composite Actions and Reusable Workflows +are the following: + +- Composite Actions can be executed from a remote/external path or from the checked out branch, and from any location. + However, Reusable Workflows can only be used through a remote/external path (`{owner}/{repo}/{path}/{filename}@{ref}`), + where `{path}` must be `.github/workflows`, and `@{ref}` is required. + See [actions/runner#1493](https://github.com/actions/runner/issues/1493). + As a result: + - Local Composite Actions cannot be used without a prior repo checkout, but Reusable Workflows can be used without + checkout. + - Testing development versions of local Reusable Workflows is cumbersome, because PRs do not pick the modifications by + default. +- Composite Actions can include multiple steps, but not multiple jobs. + Conversely, Reusable Workflows can include multiple jobs, and multiple steps in each job. +- Composite Actions can include multiple files, so it's possible to use files from the Action or from the user's repository. + Conversely, Reusable Workflows are a single YAML file, with no additional files retrieved by default. + +### Callable vs dispatchable workflows + +Reusable Workflows are defined through the `workflow_call` event kind. +Similarly, any "regular" Workflow can be triggered through a `workflow_dispatch` event. +Both event kinds support `input` options, which are usable within the Workflow. +Therefore, one might intuitively try to write a workflow which is both callable and dispatchable. +In other words, which can be either reused from another workflow, or triggered through the API. +Unfortunately, that is not the case. +Although `input` options can be duplicated for both events, GitHub's backend exposes them through different objects. +In dispatchable Workflows, the object is `${{ github.event.inputs }}`, while callable workflows receive `${{ inputs }}`. + +As a result, in order to make a reusable workflow dispatchable, a wrapper workflow is required. +See, for instance, [hdl/containers: .github/workflows/common.yml](https://github.com/hdl/containers/blob/main/.github/workflows/common.yml) and [hdl/containers: .github/workflows/dispatch.yml](https://github.com/hdl/containers/blob/main/.github/workflows/dispatch.yml). +Alternatively, a normalisation job might be used, similar to the `Params` in this repo. + +### Call hierarchy + +Reusable Workflows cannot call other Reusable Workflows, however, they can use Composite Actions and Composite Actions +can call other Actions. +Therefore, in some use cases it is sensible to combine one layer of reusable workflows for orchestrating the jobs, along +with multiple layers of composite actions. + +### Script with post step + +JavaScript Actions support defining `pre`, `pre-if`, `post` and `post-if` steps, which allow executing steps at the +beginning or the end of a job, regardless of intermediate steps failing. +Unfortunately, those are not available for any other Action type. + +Action [with-post-step](with-post-step) is a generic JS Action to execute a main command and to set a command as a post +step. +It allows using the `post` feature with scripts written in bash, python or any other interpreted language available on +the environment. +See: [actions/runner#1478](https://github.com/actions/runner/issues/1478). + +## Reusable workflows + +This repository provides 10+ Reusable Workflows based on the CI pipelines of the repos in this organisation, +[EDAยฒ](https://github.com/edaa-org), [VHDL](https://github.com/vhdl), and others. +By combining them, Python packages can be continuously tested and released along with Sphinx documentation sites, to GitHub Releases, GitHub Pages and PyPI. +Optionally, coverage and static type check reports can be gathered. + +[![](ExamplePipeline.png)](ExamplePipeline.png) + +As shown in the screenshot above, the expected order is: + +- Global: + - [Params](.github/workflows/Params.yml): a workaround for the limitations to handle global variables in + GitHub Actions workflows (see [actions/runner#480](https://github.com/actions/runner/issues/480)). + It generates outputs with artifact names and job matrices to be used in other jobs. +- Code testing/analysis: + - [UnitTesting](.github/workflows/UnitTesting.yml): run unit test with `pytest` using multiple versions of Python, and + optionally upload results as XML reports. + - [CoverageCollection](.github/workflows/CoverageCollection.yml): collect coverage data with `pytest` using a single + version of Python, generate HTML and Cobertura (XML) reports, upload the HTML report as an artifact, and upload the + results to Codecov and Codacy. + - [StaticTypeCheck](.github/workflows/StaticTypeCheck.yml): collect static type check result with `mypy`, and + optionally upload results as an HTML report. + Example `commands`: + + 1. Regular package + + ```yml + commands: mypy --html-report htmlmypy -p ToolName + ``` + + 2. Parent namespace package + + ```yml + commands: | + touch Parent/__init__.py + mypy --html-report htmlmypy -p ToolName + ``` + + 3. Child namespace package + + ```yml + commands: | + cd Parent + mypy --html-report ../htmlmypy -p ToolName + ``` + + - [VerifyDocs](.github/workflows/VerifyDocs.yml): extract code examples from the README and test. +- Packaging and releasing: + - [Release](.github/workflows/Release.yml): publish GitHub Release. + - [Package](.github/workflows/Package.yml): generate source and wheel packages, and upload them as an artifact. + - [PublishOnPyPI](.github/workflows/PublishOnPyPI.yml): publish source and wheel packages to PyPI. +- Documentation: + - [BuildTheDocs](.github/workflows/BuildTheDocs.yml): build Sphinx documentation with BuildTheDocs, and upload HTML as + an artifact. + - [PublishToGitHubPages](.github/workflows/PublishToGitHubPages.yml): publish HTML documentation to GitHub Pages. +- Cleanup: + - [ArtifactCleanUp](.github/workflows/ArtifactCleanUp.yml): delete artifacts. + +### Example pipeline + +[ExamplePipeline.yml](ExamplePipeline.yml) is an example Workflow which uses all of the Reusable Workflows. +Python package/tool developers can copy it into their repos, in order to use al the reusable workflows straightaway. +Minimal required modifications are the following: + +- Set the `name` input of job `Params`. +- Specify the `commands` input of job `StaticTypeCheck`. + +Find further usage cases in the following list of projects: + +- [edaa-org/pyEDAA.ProjectModel](https://github.com/edaa-org/pyEDAA.ProjectModel/tree/main/.github/workflows) +- [edaa-org/pySVModel](https://github.com/edaa-org/pySVModel/tree/main/.github/workflows) +- [VHDL/pyVHDLModel](https://github.com/VHDL/pyVHDLModel/tree/main/.github/workflows) diff --git a/with-post-step/action.yml b/with-post-step/action.yml new file mode 100644 index 0000000..560f77f --- /dev/null +++ b/with-post-step/action.yml @@ -0,0 +1,17 @@ +name: With post step +description: 'Generic JS Action to execute a main command and set a command as a post step.' +inputs: + main: + description: 'Main command/script.' + required: true + post: + description: 'Post command/script.' + required: true + key: + description: 'Name of the state variable used to detect the post step.' + required: false + default: POST +runs: + using: 'node12' + main: 'main.js' + post: 'main.js' diff --git a/with-post-step/main.js b/with-post-step/main.js new file mode 100644 index 0000000..180e396 --- /dev/null +++ b/with-post-step/main.js @@ -0,0 +1,44 @@ +// Authors: +// Unai Martinez-Corral +// +// Copyright 2021 Unai Martinez-Corral +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Context: +// * https://github.com/docker/login-action/issues/72 +// * https://github.com/actions/runner/issues/1478 + +const { exec } = require("child_process"); + +function run(cmd) { + exec(cmd, (error, stdout, stderr) => { + if ( stdout.length !== 0 ) { console.log(`${stdout}`); } + if ( stderr.length !== 0 ) { console.error(`${stderr}`); } + if (error) { + process.exitCode = error.code; + console.error(`${error}`); + } + }); +} + +const key = process.env.INPUT_KEY.toUpperCase(); + +if ( process.env[`STATE_${key}`] !== undefined ) { // Are we in the 'post' step? + run(process.env.INPUT_POST); +} else { // Otherwise, this is the main step + console.log(`::save-state name=${key}::true`); + run(process.env.INPUT_MAIN); +}