diff --git a/.github/workflows/TestReleaser.yml b/.github/workflows/TestReleaser.yml new file mode 100644 index 0000000..e9035e4 --- /dev/null +++ b/.github/workflows/TestReleaser.yml @@ -0,0 +1,133 @@ +name: Test Releaser + +on: + push: + tags: + - '*' + - '!tip' + - '!v*' + branches: + - '**' + - '!r*' + workflow_dispatch: + schedule: + - cron: '0 0 * * 4' + +env: + CI: true + +jobs: + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt + + - name: Single + uses: ./releaser + with: + rm: true + token: ${{ secrets.GITHUB_TOKEN }} + files: artifact-*.txt + + - name: List + uses: ./releaser + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact-*.txt + README.md + + - name: Add artifacts/*.txt + run: | + mkdir artifacts + echo "Build some tool and generate some artifacts" > artifacts/artifact.txt + touch artifacts/empty_file.txt + + - name: Single in subdir + uses: ./releaser + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/artifact.txt + + - name: Add artifacts/*.md + run: | + echo "releaser hello" > artifacts/hello.md + echo "releaser world" > artifacts/world.md + + - name: Directory wildcard + uses: ./releaser + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/* + + - name: Add artifacts/subdir + run: | + mkdir artifacts/subdir + echo "Test recursive glob" > artifacts/subdir/deep_file.txt + + - name: Directory wildcard (recursive) + uses: ./releaser + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/** + + + test-composite: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt + + - name: Single + uses: ./releaser/composite + with: + rm: true + token: ${{ secrets.GITHUB_TOKEN }} + files: artifact-*.txt + + - name: List + uses: ./releaser/composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact-*.txt + README.md + + - name: Add artifacts/*.txt + run: | + mkdir artifacts + echo "Build some tool and generate some artifacts" > artifacts/artifact.txt + touch artifacts/empty_file.txt + + - name: Single in subdir + uses: ./releaser/composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/artifact.txt + + - name: Add artifacts/*.md + run: | + echo "releaser hello" > artifacts/hello.md + echo "releaser world" > artifacts/world.md + + - name: Directory wildcard + uses: ./releaser/composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/* + + - name: Add artifacts/subdir + run: | + mkdir artifacts/subdir + echo "Test recursive glob" > artifacts/subdir/deep_file.txt + + - name: Directory wildcard (recursive) + uses: ./releaser/composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/** diff --git a/releaser/Dockerfile b/releaser/Dockerfile new file mode 100644 index 0000000..2290f49 --- /dev/null +++ b/releaser/Dockerfile @@ -0,0 +1,4 @@ +FROM python:3.9-slim-bullseye +COPY releaser.py /releaser.py +RUN pip install PyGithub --progress-bar off +CMD ["/releaser.py"] diff --git a/releaser/README.md b/releaser/README.md new file mode 100644 index 0000000..703e65f --- /dev/null +++ b/releaser/README.md @@ -0,0 +1,186 @@ +# Releaser + +**Releaser** is a Docker GitHub Action written in Python. + +**Releaser** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds. +Combined with a workflow that is executed periodically, **Releaser** allows to provide a fixed release name for users willing +to use daily/nightly artifacts of a project. + +Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Releaser** can create a release and +upload assets. + +## Context + +GitHub provides official clients for the GitHub API through [github.com/octokit](https://github.com/octokit): + +- [octokit.js](https://github.com/octokit/octokit.js) ([octokit.github.io/rest.js](https://octokit.github.io/rest.js)) +- [octokit.rb](https://github.com/octokit/octokit.rb) ([octokit.github.io/octokit.rb](http://octokit.github.io/octokit.rb)) +- [octokit.net](https://github.com/octokit/octokit.net) ([octokitnet.rtfd.io](https://octokitnet.rtfd.io)) + +When GitHub Actions was released in 2019, two Actions were made available through [github.com/actions](https://github.com/actions) for dealing with GitHub Releases: + +- [actions/create-release](https://github.com/actions/create-release) +- [actions/upload-release-asset](https://github.com/actions/upload-release-asset) + +However, those Actions were contributed by an employee in spare time, not officially supported by GitHub. +Therefore, they were unmaintained before GitHub Actions was out of the private beta (see [actions/upload-release-asset#58](https://github.com/actions/upload-release-asset/issues/58)) and, a year later, archived. +Those Actions are based on [actions/toolkit](https://github.com/actions/toolkit)'s hydrated version of octokit.js. + +From a practical point of view, [actions/github-script](https://github.com/actions/github-script) is the natural replacement to those Actions, since it allows to use a pre-authenticated octokit.js client along with the workflow run context. +Still, it requires writing plain JavaScript. + +Alternatively, there are non-official GitHub API libraries available in other languages (see [docs.github.com: rest/overview/libraries](https://docs.github.com/en/rest/overview/libraries)). +**Releaser** is based on [PyGithub/PyGithub](https://github.com/PyGithub/PyGithub), a Python client for the GitHub API. + +**Releaser** was originally created in [eine/tip](https://github.com/eine/tip), as an enhanced alternative to using +`actions/create-release` and `actions/upload-release-asset`, in order to cover certain use cases that were being +migrated from Travis CI to GitHub Actions. +The main limitation of GitHub's Actions was/is verbosity and not being possible to dynamically define the list of assets +to be uploaded. + +On the other hand, GitHub Actions artifacts do require login in order to download them. +Conversely, assets of GitHub Releases can be downloaded without login. +Therefore, in order to make CI results available to the widest audience, some projects prefer having tarballs available +as assets. +In this context, one of the main use cases of **Releaser** is pushing artifacts as release assets. +Thus, the name of the Action. + +## Usage + +The following block shows a minimal YAML workflow file: + +```yml +name: 'workflow' + +on: + schedule: + - cron: '0 0 * * 5' + +jobs: + mwe: + runs-on: ubuntu-latest + steps: + + # Clone repository + - uses: actions/checkout@v2 + + # Build your application, tool, artifacts, etc. + - name: Build + run: | + echo "Build some tool and generate some artifacts" > artifact.txt + + # Update tag and pre-release + # - Update (force-push) tag to the commit that is used in the workflow. + # - Upload artifacts defined by the user. + - uses: pyTooling/Actions/releaser@main + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact.txt + README.md +``` + +### Troubleshooting + +GitHub's internal connections seem not to be very stable; as a result, uploading artifacts as assets does produce +failures rather frequently, particularly if large tarballs are to be published. +When failures are produced, some assets are left in a broken state within the release. +**Releaser** tries to handle those cases by first uploading assets with a `tmp.*` name and then renaming them; if an existing +`tmp.*` is found, it is removed and the upload is retried. +Therefore, restarting the **Releaser** job should suffice for "fixing" a failing run. + +Note: + Currently, GitHub Actions does not allow restarting a single job. + That is unfortunate, because **Releaser** is typically used as the last dependent job in the workflows. + Hence, running **Releaser** again requires restarting the whole workflow. + Fortunately, restarting individual jobs is expected to be supported on GitHub Actions in the future. + See [github/roadmap#271](https://github.com/github/roadmap/issues/271) and [actions/runner#432](https://github.com/actions/runner/issues/432). + +If the tip/nightly release generated with **Releaser** is broken, and restarting the run cannot fix it, the recommended +procedure is the following: + +1. Go to `https://github.com///releases/edit/`. +2. Edit the assets to: + - Remove the ones with a warning symbol and/or named starting with `tmp.*`. + - Or, remove all of them. +3. Save the changes (click the `Update release` button) and restart the **Releaser** job in CI. +5. If that does still not work, remove the release and restart the **Releaser** job in CI. + +See also [eine/tip#160](https://github.com/eine/tip/issues/160). + +Note: + If all the assets are removed, or if the release itself is removed, tip/nightly assets won't be available for + users until the workflow is successfully run. + For instance, Action [setup-ghdl-ci](https://github.com/ghdl/setup-ghdl-ci) uses assets from [ghdl/ghdl: releases/tag/nightly](https://github.com/ghdl/ghdl/releases/tag/nightly). + Hence, it is recommended to try removing the conflictive assets only, in order to maximise the availability. + +### Composite Action + +The default implementation of **Releaser** is a Container Action. +Therefore, in each run, the container image is built before starting the job. +Alternatively, a Composite Action version is available: `uses: pyTooling/Actions/releaser/composite@main`. +The Composite version installs the dependencies on the host (the runner environment), instead of using a container. +Both implementations are functionally equivalent from **Releaser**'s point of view; however, the Composite Action allows users +to tweak the version of Python by using [actions/setup-python](https://github.com/actions/setup-python) before. + +## Options + +All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` and/or `INPUT_SNAPSHOTS`. + +### token (required) + +Token to make authenticated API calls; can be passed in using `{{ secrets.GITHUB_TOKEN }}`. + +### files (required) + +Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the hierarchy. + +For creating/updating a release without uploading assets, set `files: none`. + +### tag + +The default tag name for the tip/nightly pre-release is `tip`, but it can be optionally overriden through option `tag`. + +### rm + +Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). Otherwise (by default), all previours artifacts are preserved or overwritten. + +### snapshots + +Whether to create releases from any tag or to treat some as snapshots. By default, all the tags with non-empty `prerelease` field (see [semver.org: Is there a suggested regular expression (RegEx) to check a SemVer string?](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string)) are considered snapshots; neither a release is created nor assets are uploaded. + +## Advanced/complex use cases + +**Releaser** is essentially a very fine wrapper to use the GitHub Actions context data along with the classes +and methods of PyGithub. + +Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements might find it desirable to write their own Python script, instead of using **Releaser**. +In fact, since `shell: python` is supported in GitHub Actions, using Python does *not* require any Action. +For prototyping purposes, the following job might be useful: + +```yml + Release: + name: '馃摝 Release' + runs-on: ubuntu-latest + needs: + - ... + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/tags/')) + steps: + + - uses: actions/download-artifact@v2 + + - shell: bash + run: pip install PyGithub --progress-bar off + + - name: Set list of files for uploading + id: files + shell: python + run: | + from github import Github + print("路 Get GitHub API handler (authenticate)") + gh = Github('${{ github.token }}') + print("路 Get Repository handler") + gh_repo = gh.get_repo('${{ github.repository }}') +``` + +Find a non-trivial use case at [msys2/msys2-autobuild](https://github.com/msys2/msys2-autobuild). diff --git a/releaser/action.yml b/releaser/action.yml new file mode 100644 index 0000000..ee7e73a --- /dev/null +++ b/releaser/action.yml @@ -0,0 +1,24 @@ +name: 'Releaser' +description: 'Publish releases, upload assets and update tip/nightly tags' +inputs: + token: + description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}' + required: true + files: + description: 'Multi-line list of glob patterns describing the artifacts to be uploaded' + required: true + tag: + description: 'Name of the tag that corresponds to the tip/nightly pre-release' + required: false + default: tip + rm: + description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name' + required: false + default: false + snapshots: + description: 'Whether to create releases from any tag or to treat some as snapshots' + required: false + default: true +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/releaser/composite/action.yml b/releaser/composite/action.yml new file mode 100644 index 0000000..4fde76f --- /dev/null +++ b/releaser/composite/action.yml @@ -0,0 +1,36 @@ +name: 'Releaser' +description: 'Publish releases, upload assets and update tip/nightly tags' +inputs: + token: + description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}' + required: true + files: + description: 'Multi-line list of glob patterns describing the artifacts to be uploaded' + required: true + tag: + description: 'Name of the tag that corresponds to the tip/nightly pre-release' + required: false + default: tip + rm: + description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name' + required: false + default: false + snapshots: + description: 'Whether to create releases from any tag or to treat some as snapshots' + required: false + default: true +runs: + using: 'composite' + steps: + + - shell: bash + run: pip install PyGithub --progress-bar off + + - shell: bash + run: ${{ github.action_path }}/../releaser.py + env: + INPUT_TOKEN: ${{ inputs.token }} + INPUT_FILES: ${{ inputs.files }} + INPUT_TAG: ${{ inputs.tag }} + INPUT_RM: ${{ inputs.rm }} + INPUT_SNAPSHOTS: ${{ inputs.snapshots }} diff --git a/releaser/pyproject.toml b/releaser/pyproject.toml new file mode 100644 index 0000000..55ec8d7 --- /dev/null +++ b/releaser/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 120 diff --git a/releaser/releaser.py b/releaser/releaser.py new file mode 100755 index 0000000..cc0c7b1 --- /dev/null +++ b/releaser/releaser.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +import re +from sys import argv, stdout, exit as sys_exit +from os import environ, getenv +from glob import glob +from pathlib import Path +from github import Github +from github import GithubException + +print("路 Get list of artifacts to be uploaded") + +args = [] +files = [] + +if "INPUT_FILES" in environ: + args = environ["INPUT_FILES"].split() + +if len(argv) > 1: + args = args + argv[1:] + +if len(args) == 1 and args[0] == "none": + files = [] + print("! Skipping 'files' because it's set to 'none") +elif len(args) == 0: + stdout.flush() + raise (Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) +else: + for item in args: + print(f" glob({item!s}):") + for fname in [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()]: + if Path(fname).stat().st_size == 0: + print(f" - ! Skipping empty file {fname!s}") + continue + print(f" - {fname!s}") + files.append(fname) + + if len(files) < 1: + stdout.flush() + raise (Exception("Empty list of files to upload/update!")) + +print("路 Get GitHub API handler (authenticate)") + +if "GITHUB_TOKEN" in environ: + gh = Github(environ["GITHUB_TOKEN"]) +elif "INPUT_TOKEN" in environ: + gh = Github(environ["INPUT_TOKEN"]) +else: + if "GITHUB_USER" not in environ or "GITHUB_PASS" not in environ: + stdout.flush() + raise ( + Exception( + "Need credentials to authenticate! Please, provide 'GITHUB_TOKEN', 'INPUT_TOKEN', or 'GITHUB_USER' and 'GITHUB_PASS'" + ) + ) + gh = Github(environ["GITHUB_USER"], environ["GITHUB_PASS"]) + +print("路 Get Repository handler") + +if "GITHUB_REPOSITORY" not in environ: + stdout.flush() + raise (Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY")) + +gh_repo = gh.get_repo(environ["GITHUB_REPOSITORY"]) + +print("路 Get Release handler") + +tag = getenv("INPUT_TAG", "tip") + +env_tag = None +gh_ref = environ["GITHUB_REF"] +is_prerelease = True +is_draft = False + +if gh_ref[0:10] == "refs/tags/": + env_tag = gh_ref[10:] + if env_tag != tag: + rexp = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" + semver = re.search(rexp, env_tag) + if semver == None and env_tag[0] == "v": + semver = re.search(rexp, env_tag[1:]) + tag = env_tag + if semver == None: + print(f"! Could not get semver from {gh_ref!s}") + print(f"! Treat tag '{tag!s}' as a release") + is_prerelease = False + else: + if semver.group("prerelease") is None: + # is a regular semver compilant tag + is_prerelease = False + elif getenv("INPUT_SNAPSHOTS", "true") == "true": + # is semver compilant prerelease tag, thus a snapshot (we skip it) + print("! Skipping snapshot prerelease") + sys_exit() + +gh_tag = None +try: + gh_tag = gh_repo.get_git_ref(f"tags/{tag!s}") +except Exception as e: + stdout.flush() + +if gh_tag: + try: + gh_release = gh_repo.get_release(tag) + except Exception as e: + gh_release = gh_repo.create_git_release(tag, tag, "", draft=True, prerelease=is_prerelease) + is_draft = True +else: + err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!" + if "GITHUB_SHA" not in environ: + raise (Exception(err_msg)) + try: + gh_release = gh_repo.create_git_tag_and_release( + tag, "", tag, "", environ["GITHUB_SHA"], "commit", draft=True, prerelease=is_prerelease + ) + is_draft = True + except Exception as e: + raise (Exception(err_msg)) + +print("路 Cleanup and/or upload artifacts") + +artifacts = files + +assets = gh_release.get_assets() + + +def delete_asset_by_name(name): + for asset in assets: + if asset.name == name: + asset.delete_asset() + return + + +def upload_asset(artifact, name): + try: + return gh_release.upload_asset(artifact, name=name) + except GithubException as ex: + if "already_exists" in [err["code"] for err in ex.data["errors"]]: + print(f" - {name} exists already! deleting...") + delete_asset_by_name(name) + else: + print(f" - uploading failed: {ex}") + except Exception as ex: + print(f" - uploading failed: {ex}") + + print(f" - retry uploading {name}...") + return gh_release.upload_asset(artifact, name=name) + + +def replace_asset(artifacts, asset): + print(f" > {asset!s}\n {asset.name!s}:") + for artifact in artifacts: + aname = str(Path(artifact).name) + if asset.name == aname: + print(f" - uploading tmp.{aname!s}...") + new_asset = upload_asset(artifact, name=f"tmp.{aname!s}") + print(f" - removing...{aname!s}") + asset.delete_asset() + print(f" - renaming tmp.{aname!s} to {aname!s}...") + new_asset.update_asset(aname, label=aname) + artifacts.remove(artifact) + return + print(" - keep") + + +if getenv("INPUT_RM", "false") == "true": + print("路 RM set. All previous assets are being cleared...") + for asset in assets: + print(f" - {asset.name}") + asset.delete_asset() +else: + for asset in assets: + replace_asset(artifacts, asset) + +for artifact in artifacts: + print(f" > {artifact!s}:\n - uploading...") + gh_release.upload_asset(artifact) + +stdout.flush() +print("路 Update Release reference (force-push tag)") + +if is_draft: + # Unfortunately, it seems not possible to update fields 'created_at' or 'published_at'. + print(" > Update (pre-)release") + gh_release.update_release( + gh_release.title, + "" if gh_release.body is None else gh_release.body, + draft=False, + prerelease=is_prerelease, + tag_name=gh_release.tag_name, + target_commitish=gh_release.target_commitish, + ) + +if ("GITHUB_SHA" in environ) and (env_tag is None): + sha = environ["GITHUB_SHA"] + print(f" > Force-push '{tag!s}' to {sha!s}") + gh_repo.get_git_ref(f"tags/{tag!s}").edit(sha)