From 459faf880a921b35af298613cb89c30161815fc7 Mon Sep 17 00:00:00 2001 From: umarcor Date: Mon, 20 Dec 2021 06:53:20 +0100 Subject: [PATCH] releaser: use GitHub CLI by default; remove option 'use-gh-cli' --- .github/workflows/TestReleaser.yml | 4 - releaser/README.md | 78 +++------ releaser/action.yml | 4 - releaser/composite/action.yml | 5 - releaser/releaser.py | 244 ++++++++++------------------- 5 files changed, 110 insertions(+), 225 deletions(-) diff --git a/.github/workflows/TestReleaser.yml b/.github/workflows/TestReleaser.yml index a4714aa..35c7d69 100644 --- a/.github/workflows/TestReleaser.yml +++ b/.github/workflows/TestReleaser.yml @@ -77,7 +77,6 @@ jobs: uses: ./releaser/composite with: token: ${{ secrets.GITHUB_TOKEN }} - use-gh-cli: true files: | artifact-*.txt README.md @@ -138,7 +137,6 @@ jobs: uses: ./releaser with: token: ${{ secrets.GITHUB_TOKEN }} - use-gh-cli: true files: | artifact-*.txt README.md @@ -176,5 +174,3 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/** - - diff --git a/releaser/README.md b/releaser/README.md index 4ff6d9a..4949b53 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -6,8 +6,8 @@ Combined with a workflow that is executed periodically, **Releaser** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. -Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Releaser** can create a release and -upload assets. +Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Releaser** can create a release +and upload assets. ## Context @@ -17,13 +17,16 @@ GitHub provides official clients for the GitHub API through [github.com/octokit] - [octokit.rb](https://github.com/octokit/octokit.rb) ([octokit.github.io/octokit.rb](http://octokit.github.io/octokit.rb)) - [octokit.net](https://github.com/octokit/octokit.net) ([octokitnet.rtfd.io](https://octokitnet.rtfd.io)) -When GitHub Actions was released in 2019, two Actions were made available through [github.com/actions](https://github.com/actions) for dealing with GitHub Releases: +When GitHub Actions was released in 2019, two Actions were made available through +[github.com/actions](https://github.com/actions) for dealing with GitHub Releases: - [actions/create-release](https://github.com/actions/create-release) - [actions/upload-release-asset](https://github.com/actions/upload-release-asset) However, those Actions were contributed by an employee in spare time, not officially supported by GitHub. -Therefore, they were unmaintained before GitHub Actions was out of the private beta (see [actions/upload-release-asset#58](https://github.com/actions/upload-release-asset/issues/58)) and, a year later, archived. +Therefore, they were unmaintained before GitHub Actions was out of the private beta +(see [actions/upload-release-asset#58](https://github.com/actions/upload-release-asset/issues/58)) +and, a year later, archived. Those Actions are based on [actions/toolkit](https://github.com/actions/toolkit)'s hydrated version of octokit.js. From a practical point of view, [actions/github-script](https://github.com/actions/github-script) is the natural replacement to those Actions, since it allows to use a pre-authenticated *octokit.js* client along with the workflow run context. @@ -80,52 +83,19 @@ jobs: README.md ``` -### Troubleshooting - -GitHub's internal connections seem not to be very stable; as a result, uploading artifacts as assets does produce -failures rather frequently, particularly if large tarballs are to be published. -When failures are produced, some assets are left in a broken state within the release. -**Releaser** tries to handle those cases by first uploading assets with a `tmp.*` name and then renaming them; if an existing -`tmp.*` is found, it is removed and the upload is retried. -Therefore, restarting the **Releaser** job should suffice for "fixing" a failing run. - -Note: - Currently, GitHub Actions does not allow restarting a single job. - That is unfortunate, because **Releaser** is typically used as the last dependent job in the workflows. - Hence, running **Releaser** again requires restarting the whole workflow. - Fortunately, restarting individual jobs is expected to be supported on GitHub Actions in the future. - See [github/roadmap#271](https://github.com/github/roadmap/issues/271) and [actions/runner#432](https://github.com/actions/runner/issues/432). - -If the tip/nightly release generated with **Releaser** is broken, and restarting the run cannot fix it, the recommended -procedure is the following: - -1. Go to `https://github.com///releases/edit/`. -2. Edit the assets to: - - Remove the ones with a warning symbol and/or named starting with `tmp.*`. - - Or, remove all of them. -3. Save the changes (click the `Update release` button) and restart the **Releaser** job in CI. -5. If that does still not work, remove the release and restart the **Releaser** job in CI. - -See also [eine/tip#160](https://github.com/eine/tip/issues/160). - -Note: - If all the assets are removed, or if the release itself is removed, tip/nightly assets won't be available for - users until the workflow is successfully run. - For instance, Action [setup-ghdl-ci](https://github.com/ghdl/setup-ghdl-ci) uses assets from [ghdl/ghdl: releases/tag/nightly](https://github.com/ghdl/ghdl/releases/tag/nightly). - Hence, it is recommended to try removing the conflictive assets only, in order to maximise the availability. - ### Composite Action The default implementation of **Releaser** is a Container Action. Therefore, a pre-built container image is pulled before starting the job. Alternatively, a Composite Action version is available: `uses: pyTooling/Actions/releaser/composite@main`. The Composite version installs the dependencies on the host (the runner environment), instead of using a container. -Both implementations are functionally equivalent from **Releaser**'s point of view; however, the Composite Action allows users -to tweak the version of Python by using [actions/setup-python](https://github.com/actions/setup-python) before. +Both implementations are functionally equivalent from **Releaser**'s point of view; however, the Composite Action allows +users to tweak the version of Python by using [actions/setup-python](https://github.com/actions/setup-python) before. ## Options -All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` and/or `INPUT_SNAPSHOTS`. +All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` +and/or `INPUT_SNAPSHOTS`. ### token (required) @@ -133,7 +103,8 @@ Token to make authenticated API calls; can be passed in using `{{ secrets.GITHUB ### files (required) -Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the hierarchy. +Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the +hierarchy. For creating/updating a release without uploading assets, set `files: none`. @@ -143,25 +114,28 @@ The default tag name for the tip/nightly pre-release is `tip`, but it can be opt ### rm -Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). Otherwise (by default), all previours artifacts are preserved or overwritten. +Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). +Otherwise (by default), all previours artifacts are preserved or overwritten. + +Note: + If all the assets are removed, or if the release itself is removed, tip/nightly assets won't be available for + users until the workflow is successfully run. + For instance, Action [setup-ghdl-ci](https://github.com/ghdl/setup-ghdl-ci) uses assets from [ghdl/ghdl: releases/tag/nightly](https://github.com/ghdl/ghdl/releases/tag/nightly). + Hence, it is recommended to try removing the conflictive assets only, in order to maximise the availability. ### snapshots -Whether to create releases from any tag or to treat some as snapshots. By default, all the tags with non-empty `prerelease` field (see [semver.org: Is there a suggested regular expression (RegEx) to check a SemVer string?](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string)) are considered snapshots; neither a release is created nor assets are uploaded. - -### use-gh-cli - -In order to work around the reliability issues explained in section *Troubleshooting* above, option *use-gh-cli* allows -using GitHub's official command line tool ([cli/cli](https://github.com/cli/cli)) for uploading/updating assets. - -IMPORTANT: Using this option requires the repository to be cloned (preferredly through [actions/checkout](https://github.com/actions/checkout)). +Whether to create releases from any tag or to treat some as snapshots. +By default, all the tags with non-empty `prerelease` field (see [semver.org: Is there a suggested regular expression (RegEx) to check a SemVer string?](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string)) +are considered snapshots; neither a release is created nor assets are uploaded. ## Advanced/complex use cases **Releaser** is essentially a very thin wrapper to use the GitHub Actions context data along with the classes and methods of PyGithub. -Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements might find it desirable to write their own Python script, instead of using **Releaser**. +Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements +might find it desirable to write their own Python script, instead of using **Releaser**. In fact, since `shell: python` is supported in GitHub Actions, using Python does *not* require any Action. For prototyping purposes, the following job might be useful: diff --git a/releaser/action.yml b/releaser/action.yml index 3ae3fba..b212e6f 100644 --- a/releaser/action.yml +++ b/releaser/action.yml @@ -40,10 +40,6 @@ inputs: description: 'Whether to create releases from any tag or to treat some as snapshots' required: false default: true - use-gh-cli: - description: 'Whether to use the GitHub CLI for uploading artifacts (requires actions/checkout)' - required: false - default: false runs: using: 'docker' image: 'docker://ghcr.io/pytooling/releaser' diff --git a/releaser/composite/action.yml b/releaser/composite/action.yml index f018513..b1a0599 100644 --- a/releaser/composite/action.yml +++ b/releaser/composite/action.yml @@ -40,10 +40,6 @@ inputs: description: 'Whether to create releases from any tag or to treat some as snapshots' required: false default: true - use-gh-cli: - description: 'Whether to use the GitHub CLI for uploading artifacts (requires actions/checkout)' - required: false - default: false runs: using: 'composite' steps: @@ -59,4 +55,3 @@ runs: INPUT_TAG: ${{ inputs.tag }} INPUT_RM: ${{ inputs.rm }} INPUT_SNAPSHOTS: ${{ inputs.snapshots }} - INPUT_USE-GH-CLI: ${{ inputs.use-gh-cli }} diff --git a/releaser/releaser.py b/releaser/releaser.py index c615614..710269e 100755 --- a/releaser/releaser.py +++ b/releaser/releaser.py @@ -33,14 +33,13 @@ from subprocess import check_call paramTag = getenv("INPUT_TAG", "tip") paramFiles = getenv("INPUT_FILES", None).split() paramRM = getenv("INPUT_RM", "false") == "true" -paramSnapshots = getenv("INPUT_SNAPSHOTS", "true").lower() == 'true' -paramUseGitHubCLI = getenv("INPUT_USE-GH-CLI", "false").lower() == 'true' +paramSnapshots = getenv("INPUT_SNAPSHOTS", "true").lower() == "true" paramToken = ( environ["GITHUB_TOKEN"] - if "GITHUB_TOKEN" in environ else - environ["INPUT_TOKEN"] - if "INPUT_TOKEN" in environ else - None + if "GITHUB_TOKEN" in environ + else environ["INPUT_TOKEN"] + if "INPUT_TOKEN" in environ + else None ) paramRepo = getenv("GITHUB_REPOSITORY", None) paramRef = getenv("GITHUB_REF", None) @@ -78,145 +77,70 @@ def GetGitHubAPIHandler(token): print("· Get GitHub API handler (authenticate)") if token is not None: return Github(token) - raise ( - Exception( - "Need credentials to authenticate! Please, provide 'GITHUB_TOKEN' or 'INPUT_TOKEN'" - ) - ) + raise (Exception("Need credentials to authenticate! Please, provide 'GITHUB_TOKEN' or 'INPUT_TOKEN'")) -def GetReleaseHandler(gh, repo, ref, tag, sha, snapshots): - def CheckRefSemVer(gh_ref, tag, snapshots): - print("· Check SemVer compliance of the reference/tag") - env_tag = None - if gh_ref[0:10] == "refs/tags/": - env_tag = gh_ref[10:] - if env_tag != tag: - rexp = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" - semver = re.search(rexp, env_tag) - if semver == None and env_tag[0] == "v": - semver = re.search(rexp, env_tag[1:]) - tag = env_tag - if semver == None: - print(f"! Could not get semver from {gh_ref!s}") - print(f"! Treat tag '{tag!s}' as a release") - return (tag, env_tag, False) - else: - if semver.group("prerelease") is None: - # is a regular semver compilant tag - return (tag, env_tag, False) - elif snapshots: - # is semver compilant prerelease tag, thus a snapshot (we skip it) - print("! Skipping snapshot prerelease") - sys_exit() - - return (tag, env_tag, True) - - def GetRepositoryHandler(repo): - print("· Get Repository handler") - if repo is None: - stdout.flush() - raise (Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY")) - return gh.get_repo(repo) - - def GetOrCreateRelease(gh_repo, tag, sha, is_prerelease): - print("· Get Release handler") - gh_tag = None - try: - gh_tag = gh_repo.get_git_ref(f"tags/{tag!s}") - except Exception: - stdout.flush() - - if gh_tag: - try: - return (gh_repo.get_release(tag), False) - except Exception: - return (gh_repo.create_git_release(tag, tag, "", draft=True, prerelease=is_prerelease), True) - else: - err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!" - if sha is None: - raise (Exception(err_msg)) - try: - return ( - gh_repo.create_git_tag_and_release( - tag, "", tag, "", sha, "commit", draft=True, prerelease=is_prerelease - ), - True, - ) - except Exception: - raise (Exception(err_msg)) - - [tag, env_tag, is_prerelease] = CheckRefSemVer(ref, tag, snapshots) - gh_repo = GetRepositoryHandler(repo) - [gh_release, is_draft] = GetOrCreateRelease(gh_repo, tag, sha, is_prerelease) - - return (gh_repo, gh_release, tag, env_tag, is_prerelease, is_draft) - - -def UploadArtifacts(gh_release, artifacts, remove, token, UseGitHubCLI): - print("· Cleanup and/or upload artifacts") - - assets = gh_release.get_assets() - - def delete_all_assets(assets): - print("· RM set. All previous assets are being cleared...") - for asset in assets: - print(f" - {asset.name}") - asset.delete_asset() - - def delete_asset_by_name(name): - for asset in assets: - if asset.name == name: - asset.delete_asset() - return - - def upload_asset(artifact, name): - try: - return gh_release.upload_asset(artifact, name=name) - except GithubException as ex: - if "already_exists" in [err["code"] for err in ex.data["errors"]]: - print(f" - {name} exists already! deleting...") - delete_asset_by_name(name) +def CheckRefSemVer(gh_ref, tag, snapshots): + print("· Check SemVer compliance of the reference/tag") + env_tag = None + if gh_ref[0:10] == "refs/tags/": + env_tag = gh_ref[10:] + if env_tag != tag: + rexp = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" + semver = re.search(rexp, env_tag) + if semver == None and env_tag[0] == "v": + semver = re.search(rexp, env_tag[1:]) + tag = env_tag + if semver == None: + print(f"! Could not get semver from {gh_ref!s}") + print(f"! Treat tag '{tag!s}' as a release") + return (tag, env_tag, False) else: - print(f" - uploading failed: {ex}") - except Exception as ex: - print(f" - uploading failed: {ex}") + if semver.group("prerelease") is None: + # is a regular semver compilant tag + return (tag, env_tag, False) + elif snapshots: + # is semver compilant prerelease tag, thus a snapshot (we skip it) + print("! Skipping snapshot prerelease") + sys_exit() - print(f" - retry uploading {name}...") - return gh_release.upload_asset(artifact, name=name) + return (tag, env_tag, True) - def replace_asset(artifacts, asset): - print(f" > {asset!s}\n {asset.name!s}:") - for artifact in artifacts: - aname = str(Path(artifact).name) - if asset.name == aname: - print(f" - uploading tmp.{aname!s}...") - new_asset = upload_asset(artifact, name=f"tmp.{aname!s}") - print(f" - removing...{aname!s}") - asset.delete_asset() - print(f" - renaming tmp.{aname!s} to {aname!s}...") - new_asset.update_asset(aname, label=aname) - artifacts.remove(artifact) - return - print(" - keep") - if remove: - delete_all_assets(assets) +def GetRepositoryHandler(gh, repo): + print("· Get Repository handler") + if repo is None: + stdout.flush() + raise (Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY")) + return gh.get_repo(repo) + + +def GetOrCreateRelease(gh_repo, tag, sha, is_prerelease): + print("· Get Release handler") + gh_tag = None + try: + gh_tag = gh_repo.get_git_ref(f"tags/{tag!s}") + except Exception: + stdout.flush() + + if gh_tag: + try: + return (gh_repo.get_release(tag), False) + except Exception: + return (gh_repo.create_git_release(tag, tag, "", draft=True, prerelease=is_prerelease), True) else: - if not UseGitHubCLI: - for asset in assets: - replace_asset(artifacts, asset) - - if UseGitHubCLI: - env = environ.copy() - env["GITHUB_TOKEN"] = token - cmd = ["gh", "release", "upload", "--clobber", tag] + artifacts - print(f" > {' '.join(cmd)}") - check_call(cmd, env=env) - else: - for artifact in artifacts: - print(f" > {artifact!s}:\n - uploading...") - gh_release.upload_asset(artifact) + err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!" + if sha is None: + raise (Exception(err_msg)) + try: + return ( + gh_repo.create_git_tag_and_release( + tag, "", tag, "", sha, "commit", draft=True, prerelease=is_prerelease + ), + True, + ) + except Exception: + raise (Exception(err_msg)) def UpdateReference(gh_release, tag, sha, is_prerelease, is_draft): @@ -240,27 +164,27 @@ def UpdateReference(gh_release, tag, sha, is_prerelease, is_draft): files = GetListOfArtifacts(sys_argv, paramFiles) -[gh_repo, gh_release, tag, env_tag, is_prerelease, is_draft] = GetReleaseHandler( - GetGitHubAPIHandler(paramToken), - paramRepo, - paramRef, - paramTag, - paramSHA, - paramSnapshots -) stdout.flush() -UploadArtifacts( - gh_release, - files, - paramRM, - paramToken, - paramUseGitHubCLI -) +[tag, env_tag, is_prerelease] = CheckRefSemVer(paramRef, paramTag, paramSnapshots) stdout.flush() -UpdateReference( - gh_release, - tag, - paramSHA if env_tag is None else None, - is_prerelease, - is_draft -) +gh_repo = GetRepositoryHandler(GetGitHubAPIHandler(paramToken), paramRepo) +stdout.flush() +[gh_release, is_draft] = GetOrCreateRelease(gh_repo, tag, paramSHA, is_prerelease) +stdout.flush() + +if paramRM: + print("· RM set. All previous assets are being cleared...") + for asset in gh_release.get_assets(): + print(f" - {asset.name}") + asset.delete_asset() +stdout.flush() + +print("· Cleanup and/or upload artifacts") +env = environ.copy() +env["GITHUB_TOKEN"] = paramToken +cmd = ["gh", "release", "upload", "--repo", paramRepo, "--clobber", tag] + files +print(f" > {' '.join(cmd)}") +check_call(cmd, env=env) +stdout.flush() + +UpdateReference(gh_release, tag, paramSHA if env_tag is None else None, is_prerelease, is_draft)