From c9d3643b20e96ab5d67d781b0071fdafa295f0bc Mon Sep 17 00:00:00 2001 From: eine Date: Thu, 14 May 2020 16:58:15 +0200 Subject: [PATCH 01/39] initial commit --- .github/workflows/push.yml | 29 +++++++++++ Dockerfile | 4 ++ README.md | 38 ++++++++++++++ action.yml | 16 ++++++ tip.py | 102 +++++++++++++++++++++++++++++++++++++ 5 files changed, 189 insertions(+) create mode 100644 .github/workflows/push.yml create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 action.yml create mode 100755 tip.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..b0ebc6e --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,29 @@ +name: 'tip' + +on: + push: + tags: + - '*' + - '!tip' + branches: + - '*' + pull_request: + +env: + CI: true + +jobs: + + tip: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build + run: | + echo "Build some tool and generate some artifacts" > artifact.txt + - uses: ./ + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact.txt + README.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..82ca70d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM python:alpine +COPY tip.py /tip.py +RUN pip install PyGithub --progress-bar off +ENTRYPOINT ["/tip.py"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..fb12346 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +**tip** is a Docker GitHub Action written in Python. **tip** allows to keep a pre-release and its artifacts up to date with a latest builds. Combined with a workflow that is executed periodically, **tip** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. + +The following block shows a minimal YAML workflow file: + +```yml +name: 'workflow' + +on: + schedule: + - cron: '0 0 * * 5' + +jobs: + mwe: + runs-on: ubuntu-latest + steps: + + # Clone repository + - uses: actions/checkout@v2 + + # Build your application, tool, artifacts, etc. + - name: Build + run: | + echo "Build some tool and generate some artifacts" > artifact.txt + + # Update tag and pre-release + # - Update (force-push) tag to the commit that is used in the workflow. + # - Upload artifacts defined by the user. + - uses: eine/tip@master + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact.txt + README.md +``` + +Note that the tag and the pre-release need to be created manually the first time. The workflow above will fail if the release does not exist. + +The default tag name is `tip`, but it can be optionally overriden through option `tag` or setting envvar `INPUT_TAG`. diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..ec6b715 --- /dev/null +++ b/action.yml @@ -0,0 +1,16 @@ +name: 'tip' +description: "keep a pre-release always up-to-date" +inputs: + token: + description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}' + required: true + files: + description: 'Multi-line list of glob patterns describing the artifacts to be uploaded' + required: true + tag: + description: 'Name of the tag that corresponds to the tip/nightly pre-release' + required: false + default: tip +runs: + using: 'docker' + image: 'Dockerfile' diff --git a/tip.py b/tip.py new file mode 100755 index 0000000..92c24ee --- /dev/null +++ b/tip.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +from os import environ, getenv +from sys import argv, stdout +from github import Github +from subprocess import check_call +from glob import glob + +print("· Get list of artifacts to be uploaded") + +args = [] +files = [] + +if 'INPUT_FILES' in environ: + args = environ['INPUT_FILES'].split() + +if len(argv) > 1: + args = args + argv[1:] + +if len(args) == 0: + stdout.flush() + raise(Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) + +for item in args: + items = glob(item) + print("glob(%s)" % item, "->", items) + files = files + items + +if len(files) < 1: + stdout.flush() + raise(Exception('Empty list of files to upload/update!')) + +print("· Get GitHub API handler (authenticate)") + +if 'GITHUB_TOKEN' in environ: + gh = Github(environ["GITHUB_TOKEN"]) +elif 'INPUT_TOKEN' in environ: + gh = Github(environ["INPUT_TOKEN"]) +else: + if 'GITHUB_USER' not in environ or 'GITHUB_PASS' not in environ: + stdout.flush() + raise(Exception("Need credentials to authenticate! Please, provide 'GITHUB_TOKEN', 'INPUT_TOKEN', or 'GITHUB_USER' and 'GITHUB_PASS'")) + gh = Github(environ["GITHUB_USER"], environ["GITHUB_PASS"]) + +print("· Get Repository handler") + +if 'GITHUB_REPOSITORY' not in environ: + stdout.flush() + raise(Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY")) + +gh_repo = gh.get_repo(environ['GITHUB_REPOSITORY']) + +print("· Get Release handler") + +tag = getenv('INPUT_TAG', 'tip') + +try: + gh_tag = gh_repo.get_git_ref('tags/%s' % tag) +except Exception as e: + stdout.flush() + # TODO: create the tag/release, instead of raising an exception + raise(Exception("Tag '%s' does not exist!" % tag)) + +gh_release = gh_repo.get_release(tag) + +print("· Upload artifacts") + +artifacts = files + +for asset in gh_release.get_assets(): + print(">", asset) + print(" ", asset.name) + for fname in artifacts: + if asset.name == fname: + print(" removing '%s'..." % asset.name) + asset.delete_asset() + print(" uploading '%s'..." % fname) + gh_release.upload_asset(fname) + artifacts.remove(fname) + break + +for fname in artifacts: + print(" uploading '%s'..." % fname) + gh_release.upload_asset(fname) + +stdout.flush() +print("· Update Release reference (force-push tag)") + +if ('GITHUB_SHA' in environ) and ('GITHUB_REF' in environ) and environ['GITHUB_REF'] != 'refs/tags/%s' % tag: + sha = environ['GITHUB_SHA'] + print("force-push '%s' to %s" % (tag, sha)) + gh_repo.get_git_ref('tags/%s' % tag).edit(sha) + + # TODO: alternatively, update the title/body of the release (while keeping the tag or not) + # gh_release.update_release( + # gh_release.title, + # gh_release.body, + # draft=False, + # prerelease=True, + # tag_name=gh_release.tag_name, + # target_commitish=gh_release.target_commitish + # ) From 65fae902fc57db49cf05da5cb7abe72f97210b9f Mon Sep 17 00:00:00 2001 From: eine Date: Fri, 15 May 2020 05:05:37 +0200 Subject: [PATCH 02/39] fix paths with subdirs --- .github/workflows/push.yml | 29 -------------------- .github/workflows/test.yml | 56 ++++++++++++++++++++++++++++++++++++++ tip.py | 20 ++++++++------ 3 files changed, 67 insertions(+), 38 deletions(-) delete mode 100644 .github/workflows/push.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml deleted file mode 100644 index b0ebc6e..0000000 --- a/.github/workflows/push.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: 'tip' - -on: - push: - tags: - - '*' - - '!tip' - branches: - - '*' - pull_request: - -env: - CI: true - -jobs: - - tip: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build - run: | - echo "Build some tool and generate some artifacts" > artifact.txt - - uses: ./ - with: - token: ${{ secrets.GITHUB_TOKEN }} - files: | - artifact.txt - README.md diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7cbfb8f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,56 @@ +name: 'test' + +on: + push: + tags: + - '*' + - '!tip' + branches: + - '*' + pull_request: + +env: + CI: true + +jobs: + + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - run: echo "Build some tool and generate some artifacts" > artifact.txt + + - name: Single + uses: ./ + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifact.txt + + - name: List + uses: ./ + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact.txt + README.md + + - run: | + mkdir artifacts + echo "Build some tool and generate some artifacts" > artifacts/artifact.txt + + - name: Single in subdir + uses: ./ + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/artifact.txt + + - run: | + echo "tip hello" > artifacts/hello.md + echo "tip world" > artifacts/world.md + + - name: Directory wildcard + uses: ./ + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/* diff --git a/tip.py b/tip.py index 92c24ee..2f4ca79 100755 --- a/tip.py +++ b/tip.py @@ -2,9 +2,10 @@ from os import environ, getenv from sys import argv, stdout -from github import Github from subprocess import check_call from glob import glob +from pathlib import Path +from github import Github print("· Get list of artifacts to be uploaded") @@ -70,18 +71,19 @@ artifacts = files for asset in gh_release.get_assets(): print(">", asset) print(" ", asset.name) - for fname in artifacts: - if asset.name == fname: + for artifact in artifacts: + aname = str(Path(artifact).name) + if asset.name == aname: print(" removing '%s'..." % asset.name) asset.delete_asset() - print(" uploading '%s'..." % fname) - gh_release.upload_asset(fname) - artifacts.remove(fname) + print(" uploading '%s'..." % artifact) + gh_release.upload_asset(artifact, name=aname) + artifacts.remove(artifact) break -for fname in artifacts: - print(" uploading '%s'..." % fname) - gh_release.upload_asset(fname) +for artifact in artifacts: + print(" uploading '%s'..." % artifact) + gh_release.upload_asset(artifact) stdout.flush() print("· Update Release reference (force-push tag)") From 6cf8de253a69570ccaf890a22bf20abd5d6dd6b4 Mon Sep 17 00:00:00 2001 From: Lucy Linder Date: Sat, 6 Jun 2020 20:22:05 +0200 Subject: [PATCH 03/39] add option 'rm' (#156) --- .github/workflows/test.yml | 9 +++++---- README.md | 2 ++ action.yml | 4 ++++ tip.py | 30 ++++++++++++++++++------------ 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cbfb8f..a414425 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,21 +18,22 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - run: echo "Build some tool and generate some artifacts" > artifact.txt + + - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt - name: Single uses: ./ with: + rm: true token: ${{ secrets.GITHUB_TOKEN }} - files: artifact.txt + files: artifact-*.txt - name: List uses: ./ with: token: ${{ secrets.GITHUB_TOKEN }} files: | - artifact.txt + artifact-*.txt README.md - run: | diff --git a/README.md b/README.md index fb12346..1fc2fec 100644 --- a/README.md +++ b/README.md @@ -36,3 +36,5 @@ jobs: Note that the tag and the pre-release need to be created manually the first time. The workflow above will fail if the release does not exist. The default tag name is `tip`, but it can be optionally overriden through option `tag` or setting envvar `INPUT_TAG`. + +If you systematically want to remove previous artifacts (e.g. old versions), set the `rm` option to true. diff --git a/action.yml b/action.yml index ec6b715..014aa7e 100644 --- a/action.yml +++ b/action.yml @@ -11,6 +11,10 @@ inputs: description: 'Name of the tag that corresponds to the tip/nightly pre-release' required: false default: tip + rm: + description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name' + required: false + default: false runs: using: 'docker' image: 'Dockerfile' diff --git a/tip.py b/tip.py index 2f4ca79..dc33e60 100755 --- a/tip.py +++ b/tip.py @@ -68,18 +68,24 @@ print("· Upload artifacts") artifacts = files -for asset in gh_release.get_assets(): - print(">", asset) - print(" ", asset.name) - for artifact in artifacts: - aname = str(Path(artifact).name) - if asset.name == aname: - print(" removing '%s'..." % asset.name) - asset.delete_asset() - print(" uploading '%s'..." % artifact) - gh_release.upload_asset(artifact, name=aname) - artifacts.remove(artifact) - break +if getenv('INPUT_RM', 'false') == 'true': + print("· RM set. All previous assets are being cleared...") + for asset in gh_release.get_assets(): + print(" ", asset.name) + asset.delete_asset() +else: + for asset in gh_release.get_assets(): + print(">", asset) + print(" ", asset.name) + for artifact in artifacts: + aname = str(Path(artifact).name) + if asset.name == aname: + print(" removing '%s'..." % asset.name) + asset.delete_asset() + print(" uploading '%s'..." % artifact) + gh_release.upload_asset(artifact, name=aname) + artifacts.remove(artifact) + break for artifact in artifacts: print(" uploading '%s'..." % artifact) From b4dbe55c266a6a04c3ca650d42f1d2188691f164 Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 6 Jun 2020 21:44:33 +0200 Subject: [PATCH 04/39] update README.md --- README.md | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1fc2fec..6418962 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ -**tip** is a Docker GitHub Action written in Python. **tip** allows to keep a pre-release and its artifacts up to date with a latest builds. Combined with a workflow that is executed periodically, **tip** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. +**tip** is a Docker GitHub Action written in Python. **tip** allows to keep a pre-release and its artifacts up to date with latest builds. Combined with a workflow that is executed periodically, **tip** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. + +# Usage The following block shows a minimal YAML workflow file: @@ -35,6 +37,22 @@ jobs: Note that the tag and the pre-release need to be created manually the first time. The workflow above will fail if the release does not exist. -The default tag name is `tip`, but it can be optionally overriden through option `tag` or setting envvar `INPUT_TAG`. +# Options -If you systematically want to remove previous artifacts (e.g. old versions), set the `rm` option to true. +All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG` and/or `INPUT_RM`. + +## token (required) + +Token to make authenticated API calls; can be passed in using `{{ secrets.GITHUB_TOKEN }}`. + +## files (required) + +Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the hierarchy. + +## tag + +The default tag name for the tip/nightly pre-release is `tip`, but it can be optionally overriden through option `tag`. + +## rm + +Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). Otherwise (by default), all previours artifacts are preserved or overwritten. From fc695b96610e7af52a7bfccaf2dd2f48773f4cb8 Mon Sep 17 00:00:00 2001 From: eine Date: Sun, 26 Jul 2020 00:54:16 +0200 Subject: [PATCH 05/39] upload, remove, rename (#157) --- tip.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tip.py b/tip.py index dc33e60..62a2451 100755 --- a/tip.py +++ b/tip.py @@ -68,27 +68,32 @@ print("· Upload artifacts") artifacts = files +assets = gh_release.get_assets() + if getenv('INPUT_RM', 'false') == 'true': print("· RM set. All previous assets are being cleared...") - for asset in gh_release.get_assets(): + for asset in assets: print(" ", asset.name) asset.delete_asset() else: - for asset in gh_release.get_assets(): - print(">", asset) - print(" ", asset.name) + for asset in assets: + print(" >", asset) + print(" %s:" % asset.name) for artifact in artifacts: aname = str(Path(artifact).name) if asset.name == aname: - print(" removing '%s'..." % asset.name) + print(" - uploading tmp...") + new_asset = gh_release.upload_asset(artifact, name='%s.tmp' % aname) + print(" - removing...") asset.delete_asset() - print(" uploading '%s'..." % artifact) - gh_release.upload_asset(artifact, name=aname) + print(" - renaming tmp...") + new_asset.update_asset(aname, label=aname) artifacts.remove(artifact) break for artifact in artifacts: - print(" uploading '%s'..." % artifact) + print(" >", artifact) + print(" - uploading...") gh_release.upload_asset(artifact) stdout.flush() From fe58d3aba3a224cac82cfd9f286e0dd03df56ce0 Mon Sep 17 00:00:00 2001 From: eine Date: Sun, 26 Jul 2020 04:14:06 +0200 Subject: [PATCH 06/39] add option 'snapshots' --- README.md | 14 ++++++++--- action.yml | 4 +++ tip.py | 71 ++++++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 67 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6418962..b91b199 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ -**tip** is a Docker GitHub Action written in Python. **tip** allows to keep a pre-release and its artifacts up to date with latest builds. Combined with a workflow that is executed periodically, **tip** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. +**tip** is a Docker GitHub Action written in Python. + +**tip** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds. Combined with a workflow that is executed periodically, **tip** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. + +Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **tip** can create a release and upload assets. # Usage @@ -35,11 +39,9 @@ jobs: README.md ``` -Note that the tag and the pre-release need to be created manually the first time. The workflow above will fail if the release does not exist. - # Options -All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG` and/or `INPUT_RM`. +All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` and/or `INPUT_SNAPSHOTS`. ## token (required) @@ -56,3 +58,7 @@ The default tag name for the tip/nightly pre-release is `tip`, but it can be opt ## rm Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). Otherwise (by default), all previours artifacts are preserved or overwritten. + +## snapshots + +Whether to create releases from any tag or to treat some as snapshots. By default, all the tags with non-empty `prerelease` field (see [semver.org: Is there a suggested regular expression (RegEx) to check a SemVer string?](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string)) are considered snapshots; neither a release is created nor assets are uploaded. diff --git a/action.yml b/action.yml index 014aa7e..07729a0 100644 --- a/action.yml +++ b/action.yml @@ -15,6 +15,10 @@ inputs: description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name' required: false default: false + snapshots: + description: 'Whether to create releases from any tag or to treat some as snapshots' + required: false + default: true runs: using: 'docker' image: 'Dockerfile' diff --git a/tip.py b/tip.py index 62a2451..a31ac7e 100755 --- a/tip.py +++ b/tip.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 -from os import environ, getenv +import re +import sys from sys import argv, stdout +from os import environ, getenv from subprocess import check_call from glob import glob from pathlib import Path @@ -55,14 +57,45 @@ print("· Get Release handler") tag = getenv('INPUT_TAG', 'tip') +env_tag = None +gh_ref = environ['GITHUB_REF'] +is_prerelease = True +is_draft = False + +if gh_ref[0:10] == 'refs/tags/': + env_tag = gh_ref[10:] + if env_tag != tag: + semver = re.search(r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", env_tag) + if semver.group('prerelease') is None: + # is a regular semver compilant tag + is_prerelease = False + tag = env_tag + elif getenv('INPUT_SNAPSHOTS', 'true') == 'true': + # is semver compilant prerelease tag, thus a snapshot (we skip it) + sys.exit() + +gh_tag = None try: gh_tag = gh_repo.get_git_ref('tags/%s' % tag) except Exception as e: stdout.flush() - # TODO: create the tag/release, instead of raising an exception - raise(Exception("Tag '%s' does not exist!" % tag)) - -gh_release = gh_repo.get_release(tag) + pass +if gh_tag: + try: + gh_release = gh_repo.get_release(tag) + except Exception as e: + gh_release = gh_repo.create_git_release(tag, tag, "", draft=True, prerelease=is_prerelease) + is_draft = True + pass +else: + err_msg = "Tag/release '%s' does not exist and could not create it!" % tag + if 'GITHUB_SHA' not in environ: + raise(Exception(err_msg)) + try: + gh_release = gh_repo.create_git_tag_and_release(tag, "", tag, "", environ['GITHUB_SHA'], 'commit', draft=True, prerelease=is_prerelease) + is_draft = True + except Exception as e: + raise(Exception(err_msg)) print("· Upload artifacts") @@ -99,17 +132,19 @@ for artifact in artifacts: stdout.flush() print("· Update Release reference (force-push tag)") -if ('GITHUB_SHA' in environ) and ('GITHUB_REF' in environ) and environ['GITHUB_REF'] != 'refs/tags/%s' % tag: - sha = environ['GITHUB_SHA'] - print("force-push '%s' to %s" % (tag, sha)) - gh_repo.get_git_ref('tags/%s' % tag).edit(sha) +if is_draft: + # Unfortunately, it seems not possible to update fields 'created_at' or 'published_at'. + print(" > Update (pre-)release") + gh_release.update_release( + gh_release.title, + "" if gh_release.body is None else gh_release.body, + draft=False, + prerelease=is_prerelease, + tag_name=gh_release.tag_name, + target_commitish=gh_release.target_commitish + ) - # TODO: alternatively, update the title/body of the release (while keeping the tag or not) - # gh_release.update_release( - # gh_release.title, - # gh_release.body, - # draft=False, - # prerelease=True, - # tag_name=gh_release.tag_name, - # target_commitish=gh_release.target_commitish - # ) +if ('GITHUB_SHA' in environ) and (env_tag is None): + sha = environ['GITHUB_SHA'] + print(" > Force-push '%s' to %s" % (tag, sha)) + gh_repo.get_git_ref('tags/%s' % tag).edit(sha) From 4faf2667a21544521d0e5d152f0575997a0b9908 Mon Sep 17 00:00:00 2001 From: eine Date: Sun, 26 Jul 2020 18:01:24 +0200 Subject: [PATCH 07/39] fix naming of tmp assets --- tip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip.py b/tip.py index a31ac7e..d792172 100755 --- a/tip.py +++ b/tip.py @@ -116,7 +116,7 @@ else: aname = str(Path(artifact).name) if asset.name == aname: print(" - uploading tmp...") - new_asset = gh_release.upload_asset(artifact, name='%s.tmp' % aname) + new_asset = gh_release.upload_asset(artifact, name='tmp.%s' % aname) print(" - removing...") asset.delete_asset() print(" - renaming tmp...") From 642b99b75d84acd64206ef74502e1363157a5bba Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 17:31:04 +0200 Subject: [PATCH 08/39] set glob recursive=True --- .github/workflows/test.yml | 19 ++++++++++++++++--- tip.py | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a414425..f2ccee5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - + - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt - name: Single @@ -36,7 +36,8 @@ jobs: artifact-*.txt README.md - - run: | + - name: Add artifacts/artifact.txt + run: | mkdir artifacts echo "Build some tool and generate some artifacts" > artifacts/artifact.txt @@ -46,7 +47,8 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/artifact.txt - - run: | + - name: Add artifacts/*.md + run: | echo "tip hello" > artifacts/hello.md echo "tip world" > artifacts/world.md @@ -55,3 +57,14 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/* + + - name: Add artifacts/subdir + run: | + mkdir artifacts/subdir + echo "Test recursive glob" > artifacts/subdir/deep_file.txt + + - name: Directory wildcard (recursive) + uses: ./ + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/** diff --git a/tip.py b/tip.py index d792172..4f1324a 100755 --- a/tip.py +++ b/tip.py @@ -25,7 +25,7 @@ if len(args) == 0: raise(Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) for item in args: - items = glob(item) + items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] print("glob(%s)" % item, "->", items) files = files + items From 9b5ac360b7d464dd3755b011d82674c48c1bebed Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 18:11:37 +0200 Subject: [PATCH 09/39] skip empty files (#159) --- .github/workflows/test.yml | 3 ++- tip.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f2ccee5..0cf1602 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,10 +36,11 @@ jobs: artifact-*.txt README.md - - name: Add artifacts/artifact.txt + - name: Add artifacts/*.txt run: | mkdir artifacts echo "Build some tool and generate some artifacts" > artifacts/artifact.txt + touch artifacts/empty_file.txt - name: Single in subdir uses: ./ diff --git a/tip.py b/tip.py index 4f1324a..acd18ac 100755 --- a/tip.py +++ b/tip.py @@ -27,7 +27,11 @@ if len(args) == 0: for item in args: items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] print("glob(%s)" % item, "->", items) - files = files + items + for fname in items: + if Path(fname).stat().st_size == 0: + print("! Skipping empty file %s" % fname) + continue + files += [fname] if len(files) < 1: stdout.flush() From 13eab642be3576646a01e51e76d29136a197260b Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 18:23:39 +0200 Subject: [PATCH 10/39] ci: fix branch name filter --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0cf1602..fc55a4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: - '*' - '!tip' branches: - - '*' + - '**' pull_request: env: From 4763c86e77d0e7c7086584c519405b1727560680 Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 18:51:35 +0200 Subject: [PATCH 11/39] fix: use non-snapshot prerelease tags --- tip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip.py b/tip.py index acd18ac..ec9b80b 100755 --- a/tip.py +++ b/tip.py @@ -70,10 +70,10 @@ if gh_ref[0:10] == 'refs/tags/': env_tag = gh_ref[10:] if env_tag != tag: semver = re.search(r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", env_tag) + tag = env_tag if semver.group('prerelease') is None: # is a regular semver compilant tag is_prerelease = False - tag = env_tag elif getenv('INPUT_SNAPSHOTS', 'true') == 'true': # is semver compilant prerelease tag, thus a snapshot (we skip it) sys.exit() From 2d1af559525737693861e09d91a28f6e381ac684 Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 18:53:04 +0200 Subject: [PATCH 12/39] print warning about skipping snapshots --- tip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tip.py b/tip.py index ec9b80b..eb9d894 100755 --- a/tip.py +++ b/tip.py @@ -76,6 +76,7 @@ if gh_ref[0:10] == 'refs/tags/': is_prerelease = False elif getenv('INPUT_SNAPSHOTS', 'true') == 'true': # is semver compilant prerelease tag, thus a snapshot (we skip it) + print("! Skipping snapshot prerelease") sys.exit() gh_tag = None From ceecb3268383779e9ea895b79f8816911e7e75d7 Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 19:52:19 +0200 Subject: [PATCH 13/39] accept semver tags starting with 'v' --- tip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tip.py b/tip.py index eb9d894..d5d5762 100755 --- a/tip.py +++ b/tip.py @@ -69,7 +69,10 @@ is_draft = False if gh_ref[0:10] == 'refs/tags/': env_tag = gh_ref[10:] if env_tag != tag: - semver = re.search(r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", env_tag) + rexp = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" + semver = re.search(rexp, env_tag) + if semver == None and env_tag[0] == 'v': + semver = re.search(rexp, env_tag[1:]) tag = env_tag if semver.group('prerelease') is None: # is a regular semver compilant tag From 4c1a1385fbfa0cf574d3f74912d9ad2183e23929 Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 10 Oct 2020 19:54:08 +0200 Subject: [PATCH 14/39] fix: do not crash if tag is not semver compliant --- tip.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tip.py b/tip.py index d5d5762..587044c 100755 --- a/tip.py +++ b/tip.py @@ -74,13 +74,16 @@ if gh_ref[0:10] == 'refs/tags/': if semver == None and env_tag[0] == 'v': semver = re.search(rexp, env_tag[1:]) tag = env_tag - if semver.group('prerelease') is None: - # is a regular semver compilant tag - is_prerelease = False - elif getenv('INPUT_SNAPSHOTS', 'true') == 'true': - # is semver compilant prerelease tag, thus a snapshot (we skip it) - print("! Skipping snapshot prerelease") - sys.exit() + if semver == None: + print('! Could not get semver from %s' % gh_ref) + else: + if semver.group('prerelease') is None: + # is a regular semver compilant tag + is_prerelease = False + elif getenv('INPUT_SNAPSHOTS', 'true') == 'true': + # is semver compilant prerelease tag, thus a snapshot (we skip it) + print("! Skipping snapshot prerelease") + sys.exit() gh_tag = None try: From 86bdfdbb1bcc90d210f02bdb81d799989ca97000 Mon Sep 17 00:00:00 2001 From: eine Date: Wed, 6 Jan 2021 07:22:18 +0100 Subject: [PATCH 15/39] treat non semver compliat tags as releases --- tip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tip.py b/tip.py index 587044c..39ef03d 100755 --- a/tip.py +++ b/tip.py @@ -76,6 +76,8 @@ if gh_ref[0:10] == 'refs/tags/': tag = env_tag if semver == None: print('! Could not get semver from %s' % gh_ref) + print("! Treat tag '%s' as a release" % tag) + is_prerelease = False else: if semver.group('prerelease') is None: # is a regular semver compilant tag From 0b4083dfda2344f5e496d8df88e3a3159de81524 Mon Sep 17 00:00:00 2001 From: eine Date: Tue, 12 Jan 2021 20:41:15 +0100 Subject: [PATCH 16/39] support not uploading assets, through 'files: none' --- README.md | 2 ++ tip.py | 34 +++++++++++++++++++--------------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b91b199..c322196 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ Token to make authenticated API calls; can be passed in using `{{ secrets.GITHUB Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the hierarchy. +For creating/updating a release without uploading assets, set `files: none`. + ## tag The default tag name for the tip/nightly pre-release is `tip`, but it can be optionally overriden through option `tag`. diff --git a/tip.py b/tip.py index 39ef03d..0794b35 100755 --- a/tip.py +++ b/tip.py @@ -20,22 +20,26 @@ if 'INPUT_FILES' in environ: if len(argv) > 1: args = args + argv[1:] -if len(args) == 0: - stdout.flush() - raise(Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) +if len(args) == 1 and args[0] == 'none': + files = [] + print("! Skipping 'files' because it's set to 'none") +else: + if len(args) == 0: + stdout.flush() + raise(Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) -for item in args: - items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] - print("glob(%s)" % item, "->", items) - for fname in items: - if Path(fname).stat().st_size == 0: - print("! Skipping empty file %s" % fname) - continue - files += [fname] + for item in args: + items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] + print("glob(%s)" % item, "->", items) + for fname in items: + if Path(fname).stat().st_size == 0: + print("! Skipping empty file %s" % fname) + continue + files += [fname] -if len(files) < 1: - stdout.flush() - raise(Exception('Empty list of files to upload/update!')) + if len(files) < 1: + stdout.flush() + raise(Exception('Empty list of files to upload/update!')) print("· Get GitHub API handler (authenticate)") @@ -110,7 +114,7 @@ else: except Exception as e: raise(Exception(err_msg)) -print("· Upload artifacts") +print("· Cleanup and/or upload artifacts") artifacts = files From 0698ef44ac7e1b18cfa9f4957c501e8b46db7f33 Mon Sep 17 00:00:00 2001 From: eine Date: Sat, 17 Jul 2021 20:25:12 +0200 Subject: [PATCH 17/39] ci: add workflow_dispatch and cron event --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc55a4e..3c29ea4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,6 +8,9 @@ on: branches: - '**' pull_request: + workflow_dispatch: + schedule: + - cron: '0 0 * * 4' env: CI: true From 257749f997d22478994f9961132640bb3d229579 Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 18 Oct 2021 01:32:17 +0200 Subject: [PATCH 18/39] use fstrings --- tip.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tip.py b/tip.py index 0794b35..63ee01b 100755 --- a/tip.py +++ b/tip.py @@ -30,10 +30,10 @@ else: for item in args: items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] - print("glob(%s)" % item, "->", items) + print(f"glob({item!s})", "->", items) for fname in items: if Path(fname).stat().st_size == 0: - print("! Skipping empty file %s" % fname) + print(f"! Skipping empty file {fname!s}") continue files += [fname] @@ -79,8 +79,8 @@ if gh_ref[0:10] == 'refs/tags/': semver = re.search(rexp, env_tag[1:]) tag = env_tag if semver == None: - print('! Could not get semver from %s' % gh_ref) - print("! Treat tag '%s' as a release" % tag) + print(f'! Could not get semver from {gh_ref!s}') + print(f"! Treat tag '{tag!s}' as a release") is_prerelease = False else: if semver.group('prerelease') is None: @@ -93,7 +93,7 @@ if gh_ref[0:10] == 'refs/tags/': gh_tag = None try: - gh_tag = gh_repo.get_git_ref('tags/%s' % tag) + gh_tag = gh_repo.get_git_ref(f'tags/{tag!s}') except Exception as e: stdout.flush() pass @@ -105,7 +105,7 @@ if gh_tag: is_draft = True pass else: - err_msg = "Tag/release '%s' does not exist and could not create it!" % tag + err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!" if 'GITHUB_SHA' not in environ: raise(Exception(err_msg)) try: @@ -127,13 +127,12 @@ if getenv('INPUT_RM', 'false') == 'true': asset.delete_asset() else: for asset in assets: - print(" >", asset) - print(" %s:" % asset.name) + print(f" > {asset!s}\n {asset.name!s}:") for artifact in artifacts: aname = str(Path(artifact).name) if asset.name == aname: print(" - uploading tmp...") - new_asset = gh_release.upload_asset(artifact, name='tmp.%s' % aname) + new_asset = gh_release.upload_asset(artifact, name=f'tmp.{aname!s}') print(" - removing...") asset.delete_asset() print(" - renaming tmp...") @@ -163,5 +162,5 @@ if is_draft: if ('GITHUB_SHA' in environ) and (env_tag is None): sha = environ['GITHUB_SHA'] - print(" > Force-push '%s' to %s" % (tag, sha)) - gh_repo.get_git_ref('tags/%s' % tag).edit(sha) + print(f" > Force-push '{tag!s}' to {sha!s}") + gh_repo.get_git_ref(f'tags/{tag!s}').edit(sha) From 090df199ac32008798c3ab794c90fe7bf3462920 Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 18 Oct 2021 01:32:56 +0200 Subject: [PATCH 19/39] add pyproject.toml --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..55ec8d7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 120 From c5d663973ff1d23acbbdd2f794064c63d2df7e87 Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 18 Oct 2021 01:33:27 +0200 Subject: [PATCH 20/39] run black --- tip.py | 66 ++++++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/tip.py b/tip.py index 63ee01b..7d8d7ef 100755 --- a/tip.py +++ b/tip.py @@ -14,19 +14,19 @@ print("· Get list of artifacts to be uploaded") args = [] files = [] -if 'INPUT_FILES' in environ: - args = environ['INPUT_FILES'].split() +if "INPUT_FILES" in environ: + args = environ["INPUT_FILES"].split() if len(argv) > 1: args = args + argv[1:] -if len(args) == 1 and args[0] == 'none': +if len(args) == 1 and args[0] == "none": files = [] print("! Skipping 'files' because it's set to 'none") else: if len(args) == 0: stdout.flush() - raise(Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) + raise (Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) for item in args: items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] @@ -39,61 +39,65 @@ else: if len(files) < 1: stdout.flush() - raise(Exception('Empty list of files to upload/update!')) + raise (Exception("Empty list of files to upload/update!")) print("· Get GitHub API handler (authenticate)") -if 'GITHUB_TOKEN' in environ: +if "GITHUB_TOKEN" in environ: gh = Github(environ["GITHUB_TOKEN"]) -elif 'INPUT_TOKEN' in environ: +elif "INPUT_TOKEN" in environ: gh = Github(environ["INPUT_TOKEN"]) else: - if 'GITHUB_USER' not in environ or 'GITHUB_PASS' not in environ: + if "GITHUB_USER" not in environ or "GITHUB_PASS" not in environ: stdout.flush() - raise(Exception("Need credentials to authenticate! Please, provide 'GITHUB_TOKEN', 'INPUT_TOKEN', or 'GITHUB_USER' and 'GITHUB_PASS'")) + raise ( + Exception( + "Need credentials to authenticate! Please, provide 'GITHUB_TOKEN', 'INPUT_TOKEN', or 'GITHUB_USER' and 'GITHUB_PASS'" + ) + ) gh = Github(environ["GITHUB_USER"], environ["GITHUB_PASS"]) print("· Get Repository handler") -if 'GITHUB_REPOSITORY' not in environ: +if "GITHUB_REPOSITORY" not in environ: stdout.flush() - raise(Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY")) + raise (Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY")) -gh_repo = gh.get_repo(environ['GITHUB_REPOSITORY']) +gh_repo = gh.get_repo(environ["GITHUB_REPOSITORY"]) print("· Get Release handler") -tag = getenv('INPUT_TAG', 'tip') +tag = getenv("INPUT_TAG", "tip") env_tag = None -gh_ref = environ['GITHUB_REF'] +gh_ref = environ["GITHUB_REF"] is_prerelease = True is_draft = False -if gh_ref[0:10] == 'refs/tags/': +if gh_ref[0:10] == "refs/tags/": env_tag = gh_ref[10:] if env_tag != tag: rexp = r"^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" semver = re.search(rexp, env_tag) - if semver == None and env_tag[0] == 'v': + if semver == None and env_tag[0] == "v": semver = re.search(rexp, env_tag[1:]) tag = env_tag if semver == None: - print(f'! Could not get semver from {gh_ref!s}') + print(f"! Could not get semver from {gh_ref!s}") print(f"! Treat tag '{tag!s}' as a release") is_prerelease = False else: - if semver.group('prerelease') is None: + if semver.group("prerelease") is None: # is a regular semver compilant tag is_prerelease = False - elif getenv('INPUT_SNAPSHOTS', 'true') == 'true': + elif getenv("INPUT_SNAPSHOTS", "true") == "true": # is semver compilant prerelease tag, thus a snapshot (we skip it) print("! Skipping snapshot prerelease") sys.exit() gh_tag = None try: - gh_tag = gh_repo.get_git_ref(f'tags/{tag!s}') + gh_tag = gh_repo.get_git_ref(f"tags/{tag!s}") except Exception as e: stdout.flush() pass @@ -106,13 +110,15 @@ if gh_tag: pass else: err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!" - if 'GITHUB_SHA' not in environ: - raise(Exception(err_msg)) + if "GITHUB_SHA" not in environ: + raise (Exception(err_msg)) try: - gh_release = gh_repo.create_git_tag_and_release(tag, "", tag, "", environ['GITHUB_SHA'], 'commit', draft=True, prerelease=is_prerelease) + gh_release = gh_repo.create_git_tag_and_release( + tag, "", tag, "", environ["GITHUB_SHA"], "commit", draft=True, prerelease=is_prerelease + ) is_draft = True except Exception as e: - raise(Exception(err_msg)) + raise (Exception(err_msg)) print("· Cleanup and/or upload artifacts") @@ -120,7 +126,7 @@ artifacts = files assets = gh_release.get_assets() -if getenv('INPUT_RM', 'false') == 'true': +if getenv("INPUT_RM", "false") == "true": print("· RM set. All previous assets are being cleared...") for asset in assets: print(" ", asset.name) @@ -132,7 +138,7 @@ else: aname = str(Path(artifact).name) if asset.name == aname: print(" - uploading tmp...") - new_asset = gh_release.upload_asset(artifact, name=f'tmp.{aname!s}') + new_asset = gh_release.upload_asset(artifact, name=f"tmp.{aname!s}") print(" - removing...") asset.delete_asset() print(" - renaming tmp...") @@ -157,10 +163,10 @@ if is_draft: draft=False, prerelease=is_prerelease, tag_name=gh_release.tag_name, - target_commitish=gh_release.target_commitish + target_commitish=gh_release.target_commitish, ) -if ('GITHUB_SHA' in environ) and (env_tag is None): - sha = environ['GITHUB_SHA'] +if ("GITHUB_SHA" in environ) and (env_tag is None): + sha = environ["GITHUB_SHA"] print(f" > Force-push '{tag!s}' to {sha!s}") - gh_repo.get_git_ref(f'tags/{tag!s}').edit(sha) + gh_repo.get_git_ref(f"tags/{tag!s}").edit(sha) From 02db3d0242d68c36ab714c4354144d4a1467fb69 Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 18 Oct 2021 01:46:53 +0200 Subject: [PATCH 21/39] improve handling of upload failures --- tip.py | 59 +++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 15 deletions(-) diff --git a/tip.py b/tip.py index 7d8d7ef..c309898 100755 --- a/tip.py +++ b/tip.py @@ -8,6 +8,7 @@ from subprocess import check_call from glob import glob from pathlib import Path from github import Github +from github import GithubException print("· Get list of artifacts to be uploaded") @@ -126,29 +127,57 @@ artifacts = files assets = gh_release.get_assets() + +def delete_asset_by_name(name): + for asset in assets: + if asset.name == name: + asset.delete_asset() + return + + +def upload_asset(artifact, name): + try: + return gh_release.upload_asset(artifact, name=name) + except GithubException as ex: + if "already_exists" in [err["code"] for err in ex.data["errors"]]: + print(f" - {name} exists already! deleting...") + delete_asset_by_name(name) + else: + print(f" - uploading failed: {ex}") + except Exception as ex: + print(f" - uploading failed: {ex}") + + print(f" - retry uploading {name}...") + return gh_release.upload_asset(artifact, name=name) + + +def replace_asset(artifacts, asset): + print(f" > {asset!s}\n {asset.name!s}:") + for artifact in artifacts: + aname = str(Path(artifact).name) + if asset.name == aname: + print(f" - uploading tmp.{aname!s}...") + new_asset = upload_asset(artifact, name=f"tmp.{aname!s}") + print(f" - removing...{aname!s}") + asset.delete_asset() + print(f" - renaming tmp.{aname!s} to {aname!s}...") + new_asset.update_asset(aname, label=aname) + artifacts.remove(artifact) + return + print(" - keep") + + if getenv("INPUT_RM", "false") == "true": print("· RM set. All previous assets are being cleared...") for asset in assets: - print(" ", asset.name) + print(f" - {asset.name}") asset.delete_asset() else: for asset in assets: - print(f" > {asset!s}\n {asset.name!s}:") - for artifact in artifacts: - aname = str(Path(artifact).name) - if asset.name == aname: - print(" - uploading tmp...") - new_asset = gh_release.upload_asset(artifact, name=f"tmp.{aname!s}") - print(" - removing...") - asset.delete_asset() - print(" - renaming tmp...") - new_asset.update_asset(aname, label=aname) - artifacts.remove(artifact) - break + replace_asset(artifacts, asset) for artifact in artifacts: - print(" >", artifact) - print(" - uploading...") + print(f" > {artifact!s}:\n - uploading...") gh_release.upload_asset(artifact) stdout.flush() From fb0c52cbfbb48cf4af565a5c4a003b91fb8e3bee Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 18 Oct 2021 02:15:52 +0200 Subject: [PATCH 22/39] improve listing of glob results --- tip.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tip.py b/tip.py index c309898..1554f39 100755 --- a/tip.py +++ b/tip.py @@ -31,11 +31,12 @@ else: for item in args: items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] - print(f"glob({item!s})", "->", items) + print(f" glob({item!s}):") for fname in items: if Path(fname).stat().st_size == 0: - print(f"! Skipping empty file {fname!s}") + print(f" - ! Skipping empty file {fname!s}") continue + print(f" - {fname!s}") files += [fname] if len(files) < 1: From 5dbb1c55c1a2967e8b36b3957bf48cbc7e7fac65 Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 15 Nov 2021 21:16:24 +0100 Subject: [PATCH 23/39] use image 'python:slim-bullseye' instead of 'python:alpine' --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82ca70d..75681b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:alpine +FROM python:slim-bullseye COPY tip.py /tip.py RUN pip install PyGithub --progress-bar off ENTRYPOINT ["/tip.py"] From 0ae50caafe8831537ad3f0d756668bdcf43d6e6b Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 15 Nov 2021 22:42:31 +0100 Subject: [PATCH 24/39] add 'composite' version of the Action --- .github/workflows/test.yml | 58 ++++++++++++++++++++++++++++++++++++++ composite/action.yml | 30 ++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 composite/action.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c29ea4..c3d4e62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,3 +72,61 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/** + + + test-composite: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt + + - name: Single + uses: ./composite + with: + rm: true + token: ${{ secrets.GITHUB_TOKEN }} + files: artifact-*.txt + + - name: List + uses: ./composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: | + artifact-*.txt + README.md + + - name: Add artifacts/*.txt + run: | + mkdir artifacts + echo "Build some tool and generate some artifacts" > artifacts/artifact.txt + touch artifacts/empty_file.txt + + - name: Single in subdir + uses: ./composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/artifact.txt + + - name: Add artifacts/*.md + run: | + echo "tip hello" > artifacts/hello.md + echo "tip world" > artifacts/world.md + + - name: Directory wildcard + uses: ./composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/* + + - name: Add artifacts/subdir + run: | + mkdir artifacts/subdir + echo "Test recursive glob" > artifacts/subdir/deep_file.txt + + - name: Directory wildcard (recursive) + uses: ./composite + with: + token: ${{ secrets.GITHUB_TOKEN }} + files: artifacts/** diff --git a/composite/action.yml b/composite/action.yml new file mode 100644 index 0000000..f33c0e5 --- /dev/null +++ b/composite/action.yml @@ -0,0 +1,30 @@ +name: 'tip' +description: "keep a pre-release always up-to-date" +inputs: + token: + description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}' + required: true + files: + description: 'Multi-line list of glob patterns describing the artifacts to be uploaded' + required: true + tag: + description: 'Name of the tag that corresponds to the tip/nightly pre-release' + required: false + default: tip + rm: + description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name' + required: false + default: false + snapshots: + description: 'Whether to create releases from any tag or to treat some as snapshots' + required: false + default: true +runs: + using: 'composite' + steps: + + - shell: bash + run: pip install PyGithub --progress-bar off + + - shell: bash + run: ${{ github.action_path }}/../tip.py From cec300fd51184c2c80721e52228db85e2bf86b80 Mon Sep 17 00:00:00 2001 From: eine Date: Mon, 15 Nov 2021 22:48:37 +0100 Subject: [PATCH 25/39] composite: map inputs to envvars explicitly (actions/runner#665) --- composite/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/composite/action.yml b/composite/action.yml index f33c0e5..255b4f6 100644 --- a/composite/action.yml +++ b/composite/action.yml @@ -28,3 +28,9 @@ runs: - shell: bash run: ${{ github.action_path }}/../tip.py + env: + INPUT_TOKEN: ${{ inputs.token }} + INPUT_FILES: ${{ inputs.files }} + INPUT_TAG: ${{ inputs.tag }} + INPUT_RM: ${{ inputs.rm }} + INPUT_SNAPSHOTS: ${{ inputs.snapshots }} From 7bc8117e1da9f404468da2f36c5cd56f3f66f1bf Mon Sep 17 00:00:00 2001 From: Christoph Reiter Date: Sat, 20 Nov 2021 11:49:40 +0100 Subject: [PATCH 26/39] docker: revert to Python 3.9 (#163) Related to #160. The errors started around the time when Python 3.10 was released, so go back to 3.9.x to see if that is related. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 75681b9..7b41545 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:slim-bullseye +FROM python:3.9-slim-bullseye COPY tip.py /tip.py RUN pip install PyGithub --progress-bar off ENTRYPOINT ["/tip.py"] From 4df89a2f6ad75bed98ba137e63a1caeb23b0401b Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 02:54:41 +0100 Subject: [PATCH 27/39] tip: prepare for merging into pyTooling/Actions --- .github/workflows/{test.yml => Tip.yml} | 2 +- Dockerfile => tip/Dockerfile | 0 README.md => tip/README.md | 0 action.yml => tip/action.yml | 0 {composite => tip/composite}/action.yml | 0 pyproject.toml => tip/pyproject.toml | 0 tip.py => tip/tip.py | 0 7 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{test.yml => Tip.yml} (99%) rename Dockerfile => tip/Dockerfile (100%) rename README.md => tip/README.md (100%) rename action.yml => tip/action.yml (100%) rename {composite => tip/composite}/action.yml (100%) rename pyproject.toml => tip/pyproject.toml (100%) rename tip.py => tip/tip.py (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/Tip.yml similarity index 99% rename from .github/workflows/test.yml rename to .github/workflows/Tip.yml index c3d4e62..a201851 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/Tip.yml @@ -1,4 +1,4 @@ -name: 'test' +name: Tip on: push: diff --git a/Dockerfile b/tip/Dockerfile similarity index 100% rename from Dockerfile rename to tip/Dockerfile diff --git a/README.md b/tip/README.md similarity index 100% rename from README.md rename to tip/README.md diff --git a/action.yml b/tip/action.yml similarity index 100% rename from action.yml rename to tip/action.yml diff --git a/composite/action.yml b/tip/composite/action.yml similarity index 100% rename from composite/action.yml rename to tip/composite/action.yml diff --git a/pyproject.toml b/tip/pyproject.toml similarity index 100% rename from pyproject.toml rename to tip/pyproject.toml diff --git a/tip.py b/tip/tip.py similarity index 100% rename from tip.py rename to tip/tip.py From aa63c214f8090479cb32580f727c2579f45ba378 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 04:37:29 +0100 Subject: [PATCH 28/39] ci/Tip: preprend subdir 'tip' to 'uses' fields --- .github/workflows/Tip.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/Tip.yml b/.github/workflows/Tip.yml index a201851..8f06c10 100644 --- a/.github/workflows/Tip.yml +++ b/.github/workflows/Tip.yml @@ -25,14 +25,14 @@ jobs: - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt - name: Single - uses: ./ + uses: ./tip with: rm: true token: ${{ secrets.GITHUB_TOKEN }} files: artifact-*.txt - name: List - uses: ./ + uses: ./tip with: token: ${{ secrets.GITHUB_TOKEN }} files: | @@ -46,7 +46,7 @@ jobs: touch artifacts/empty_file.txt - name: Single in subdir - uses: ./ + uses: ./tip with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/artifact.txt @@ -57,7 +57,7 @@ jobs: echo "tip world" > artifacts/world.md - name: Directory wildcard - uses: ./ + uses: ./tip with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/* @@ -68,7 +68,7 @@ jobs: echo "Test recursive glob" > artifacts/subdir/deep_file.txt - name: Directory wildcard (recursive) - uses: ./ + uses: ./tip with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/** @@ -83,14 +83,14 @@ jobs: - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt - name: Single - uses: ./composite + uses: ./tip/composite with: rm: true token: ${{ secrets.GITHUB_TOKEN }} files: artifact-*.txt - name: List - uses: ./composite + uses: ./tip/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: | @@ -104,7 +104,7 @@ jobs: touch artifacts/empty_file.txt - name: Single in subdir - uses: ./composite + uses: ./tip/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/artifact.txt @@ -115,7 +115,7 @@ jobs: echo "tip world" > artifacts/world.md - name: Directory wildcard - uses: ./composite + uses: ./tip/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/* @@ -126,7 +126,7 @@ jobs: echo "Test recursive glob" > artifacts/subdir/deep_file.txt - name: Directory wildcard (recursive) - uses: ./composite + uses: ./tip/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/** From c4e236e62752ca190a21c37ae31dcd6929171262 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 04:34:11 +0100 Subject: [PATCH 29/39] Tip/README: update 'uses' field in the usage example --- tip/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip/README.md b/tip/README.md index c322196..681ec09 100644 --- a/tip/README.md +++ b/tip/README.md @@ -31,7 +31,7 @@ jobs: # Update tag and pre-release # - Update (force-push) tag to the commit that is used in the workflow. # - Upload artifacts defined by the user. - - uses: eine/tip@master + - uses: pyTooling/Actions/tip@main with: token: ${{ secrets.GITHUB_TOKEN }} files: | From fbddab5a805219b96accb47fbc3a1d62c3c55aa8 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 04:01:54 +0100 Subject: [PATCH 30/39] Tip/README: add 'Context' and 'Advanced/complex use cases' --- tip/README.md | 97 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/tip/README.md b/tip/README.md index 681ec09..ceeffc9 100644 --- a/tip/README.md +++ b/tip/README.md @@ -1,10 +1,51 @@ -**tip** is a Docker GitHub Action written in Python. +# Tip -**tip** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds. Combined with a workflow that is executed periodically, **tip** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. +**Tip** is a Docker GitHub Action written in Python. -Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **tip** can create a release and upload assets. +**Tip** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds. +Combined with a workflow that is executed periodically, **Tip** allows to provide a fixed release name for users willing +to use daily/nightly artifacts of a project. -# Usage +Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Tip** can create a release and +upload assets. + +## Context + +GitHub provides official clients for the GitHub API through [github.com/octokit](https://github.com/octokit): + +- [octokit.js](https://github.com/octokit/octokit.js) ([octokit.github.io/rest.js](https://octokit.github.io/rest.js)) +- [octokit.rb](https://github.com/octokit/octokit.rb) ([octokit.github.io/octokit.rb](http://octokit.github.io/octokit.rb)) +- [octokit.net](https://github.com/octokit/octokit.net) ([octokitnet.rtfd.io](https://octokitnet.rtfd.io)) + +When GitHub Actions was released in 2019, two Actions were made available through [github.com/actions](https://github.com/actions) for dealing with GitHub Releases: + +- [actions/create-release](https://github.com/actions/create-release) +- [actions/upload-release-asset](https://github.com/actions/upload-release-asset) + +However, those Actions were contributed by an employee in spare time, not officially supported by GitHub. +Therefore, they were unmaintained before GitHub Actions was out of the private beta (see [actions/upload-release-asset#58](https://github.com/actions/upload-release-asset/issues/58)) and, a year later, archived. +Those Actions are based on [actions/toolkit](https://github.com/actions/toolkit)'s hydrated version of octokit.js. + +From a practical point of view, [actions/github-script](https://github.com/actions/github-script) is the natural replacement to those Actions, since it allows to use a pre-authenticated octokit.js client along with the workflow run context. +Still, it requires writing plain JavaScript. + +Alternatively, there are non-official GitHub API libraries available in other languages (see [docs.github.com: rest/overview/libraries](https://docs.github.com/en/rest/overview/libraries)). +**Tip** is based on [PyGithub/PyGithub](https://github.com/PyGithub/PyGithub), a Python client for the GitHub API. + +**Tip** was originally created in [eine/tip](https://github.com/eine/tip), as an enhanced alternative to using +`actions/create-release` and `actions/upload-release-asset`, in order to cover certain use cases that were being +migrated from Travis CI to GitHub Actions. +The main limitation of GitHub's Actions was/is verbosity and not being possible to dynamically define the list of assets +to be uploaded. + +On the other hand, GitHub Actions artifacts do require login in order to download them. +Conversely, assets of GitHub Releases can be downloaded without login. +Therefore, in order to make CI results available to the widest audience, some projects prefer having tarballs available +as assets. +In this context, one of the main use cases of **Tip** is pushing artifacts as release assets. +Thus, the name of the Action. + +## Usage The following block shows a minimal YAML workflow file: @@ -39,28 +80,64 @@ jobs: README.md ``` -# Options +## Options All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` and/or `INPUT_SNAPSHOTS`. -## token (required) +### token (required) Token to make authenticated API calls; can be passed in using `{{ secrets.GITHUB_TOKEN }}`. -## files (required) +### files (required) Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the hierarchy. For creating/updating a release without uploading assets, set `files: none`. -## tag +### tag The default tag name for the tip/nightly pre-release is `tip`, but it can be optionally overriden through option `tag`. -## rm +### rm Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). Otherwise (by default), all previours artifacts are preserved or overwritten. -## snapshots +### snapshots Whether to create releases from any tag or to treat some as snapshots. By default, all the tags with non-empty `prerelease` field (see [semver.org: Is there a suggested regular expression (RegEx) to check a SemVer string?](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string)) are considered snapshots; neither a release is created nor assets are uploaded. + +## Advanced/complex use cases + +**Tip** is essentially a very fine wrapper to use the GitHub Actions context data along with the classes +and methods of PyGithub. + +Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements might find it desirable to write their own Python script, instead of using **Tip**. +In fact, since `shell: python` is supported in GitHub Actions, using Python does *not* require any Action. +For prototyping purposes, the following job might be useful: + +```yml + Release: + name: '📦 Release' + runs-on: ubuntu-latest + needs: + - ... + if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/tags/')) + steps: + + - uses: actions/download-artifact@v2 + + - shell: bash + run: pip install PyGithub --progress-bar off + + - name: Set list of files for uploading + id: files + shell: python + run: | + from github import Github + print("· Get GitHub API handler (authenticate)") + gh = Github('${{ github.token }}') + print("· Get Repository handler") + gh_repo = gh.get_repo('${{ github.repository }}') +``` + +Find a non-trivial use case at [msys2/msys2-autobuild](https://github.com/msys2/msys2-autobuild). From 6704be35b01294fa119e34040c40f76016840cd0 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 04:49:37 +0100 Subject: [PATCH 31/39] Tip/README: add subsection 'Composite Action' --- tip/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tip/README.md b/tip/README.md index ceeffc9..d25639b 100644 --- a/tip/README.md +++ b/tip/README.md @@ -80,6 +80,15 @@ jobs: README.md ``` +### Composite Action + +The default implementation of **Tip** is a Container Action. +Therefore, in each run, the container image is built before starting the job. +Alternatively, a Composite Action version is available: `uses: pyTooling/Actions/tip/composite@main`. +The Composite version installs the dependencies on the host (the runner environment), instead of using a container. +Both implementations are functionally equivalent from **Tip**'s point of view; however, the Composite Action allows users +to tweak the version of Python by using [actions/setup-python](https://github.com/actions/setup-python) before. + ## Options All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` and/or `INPUT_SNAPSHOTS`. From 0283c5f6a30066c0efcaa051e18c971725fbd069 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 05:06:12 +0100 Subject: [PATCH 32/39] Tip: address codacy warnings --- tip/tip.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tip/tip.py b/tip/tip.py index 1554f39..25fdf35 100755 --- a/tip/tip.py +++ b/tip/tip.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 import re -import sys -from sys import argv, stdout +from sys import argv, stdout, exit as sys_exit from os import environ, getenv -from subprocess import check_call from glob import glob from pathlib import Path from github import Github @@ -95,21 +93,20 @@ if gh_ref[0:10] == "refs/tags/": elif getenv("INPUT_SNAPSHOTS", "true") == "true": # is semver compilant prerelease tag, thus a snapshot (we skip it) print("! Skipping snapshot prerelease") - sys.exit() + sys_exit() gh_tag = None try: gh_tag = gh_repo.get_git_ref(f"tags/{tag!s}") except Exception as e: stdout.flush() - pass + if gh_tag: try: gh_release = gh_repo.get_release(tag) except Exception as e: gh_release = gh_repo.create_git_release(tag, tag, "", draft=True, prerelease=is_prerelease) is_draft = True - pass else: err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!" if "GITHUB_SHA" not in environ: From d297c024074376567e273da600a24cd99b758488 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 15:27:10 +0100 Subject: [PATCH 33/39] Tip: use 'files.append()' instead of 'files +=' --- tip/tip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip/tip.py b/tip/tip.py index 25fdf35..8388bf7 100755 --- a/tip/tip.py +++ b/tip/tip.py @@ -35,7 +35,7 @@ else: print(f" - ! Skipping empty file {fname!s}") continue print(f" - {fname!s}") - files += [fname] + files.append(fname) if len(files) < 1: stdout.flush() From 22afac70b8e3db8bdd958be518eaeafd565af576 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 15:29:07 +0100 Subject: [PATCH 34/39] Tip: style --- tip/tip.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tip/tip.py b/tip/tip.py index 8388bf7..cc0c7b1 100755 --- a/tip/tip.py +++ b/tip/tip.py @@ -22,15 +22,13 @@ if len(argv) > 1: if len(args) == 1 and args[0] == "none": files = [] print("! Skipping 'files' because it's set to 'none") +elif len(args) == 0: + stdout.flush() + raise (Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) else: - if len(args) == 0: - stdout.flush() - raise (Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!")) - for item in args: - items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()] print(f" glob({item!s}):") - for fname in items: + for fname in [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()]: if Path(fname).stat().st_size == 0: print(f" - ! Skipping empty file {fname!s}") continue From d86368235820da1b57f8c493664b26e37e6b2a05 Mon Sep 17 00:00:00 2001 From: umarcor Date: Sat, 4 Dec 2021 20:33:18 +0100 Subject: [PATCH 35/39] Tip/Dockerfile: use CMD instead of ENTRYPOINT --- tip/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tip/Dockerfile b/tip/Dockerfile index 7b41545..3079fa5 100644 --- a/tip/Dockerfile +++ b/tip/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.9-slim-bullseye COPY tip.py /tip.py RUN pip install PyGithub --progress-bar off -ENTRYPOINT ["/tip.py"] +CMD ["/tip.py"] From add5afbf9d33616e8ae1925e71a6b3fd17abe19a Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 7 Dec 2021 04:30:11 +0100 Subject: [PATCH 36/39] rename 'tip' to 'releaser' --- .../workflows/{Tip.yml => TestReleaser.yml} | 30 +++++++++---------- {tip => releaser}/Dockerfile | 4 +-- {tip => releaser}/README.md | 28 ++++++++--------- {tip => releaser}/action.yml | 4 +-- {tip => releaser}/composite/action.yml | 6 ++-- {tip => releaser}/pyproject.toml | 0 tip/tip.py => releaser/releaser.py | 0 7 files changed, 36 insertions(+), 36 deletions(-) rename .github/workflows/{Tip.yml => TestReleaser.yml} (83%) rename {tip => releaser}/Dockerfile (59%) rename {tip => releaser}/README.md (84%) rename {tip => releaser}/action.yml (88%) rename {tip => releaser}/composite/action.yml (87%) rename {tip => releaser}/pyproject.toml (100%) rename tip/tip.py => releaser/releaser.py (100%) diff --git a/.github/workflows/Tip.yml b/.github/workflows/TestReleaser.yml similarity index 83% rename from .github/workflows/Tip.yml rename to .github/workflows/TestReleaser.yml index 8f06c10..5ed94ac 100644 --- a/.github/workflows/Tip.yml +++ b/.github/workflows/TestReleaser.yml @@ -1,4 +1,4 @@ -name: Tip +name: Test Releaser on: push: @@ -25,14 +25,14 @@ jobs: - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt - name: Single - uses: ./tip + uses: ./releaser with: rm: true token: ${{ secrets.GITHUB_TOKEN }} files: artifact-*.txt - name: List - uses: ./tip + uses: ./releaser with: token: ${{ secrets.GITHUB_TOKEN }} files: | @@ -46,18 +46,18 @@ jobs: touch artifacts/empty_file.txt - name: Single in subdir - uses: ./tip + uses: ./releaser with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/artifact.txt - name: Add artifacts/*.md run: | - echo "tip hello" > artifacts/hello.md - echo "tip world" > artifacts/world.md + echo "releaser hello" > artifacts/hello.md + echo "releaser world" > artifacts/world.md - name: Directory wildcard - uses: ./tip + uses: ./releaser with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/* @@ -68,7 +68,7 @@ jobs: echo "Test recursive glob" > artifacts/subdir/deep_file.txt - name: Directory wildcard (recursive) - uses: ./tip + uses: ./releaser with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/** @@ -83,14 +83,14 @@ jobs: - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt - name: Single - uses: ./tip/composite + uses: ./releaser/composite with: rm: true token: ${{ secrets.GITHUB_TOKEN }} files: artifact-*.txt - name: List - uses: ./tip/composite + uses: ./releaser/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: | @@ -104,18 +104,18 @@ jobs: touch artifacts/empty_file.txt - name: Single in subdir - uses: ./tip/composite + uses: ./releaser/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/artifact.txt - name: Add artifacts/*.md run: | - echo "tip hello" > artifacts/hello.md - echo "tip world" > artifacts/world.md + echo "releaser hello" > artifacts/hello.md + echo "releaser world" > artifacts/world.md - name: Directory wildcard - uses: ./tip/composite + uses: ./releaser/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/* @@ -126,7 +126,7 @@ jobs: echo "Test recursive glob" > artifacts/subdir/deep_file.txt - name: Directory wildcard (recursive) - uses: ./tip/composite + uses: ./releaser/composite with: token: ${{ secrets.GITHUB_TOKEN }} files: artifacts/** diff --git a/tip/Dockerfile b/releaser/Dockerfile similarity index 59% rename from tip/Dockerfile rename to releaser/Dockerfile index 3079fa5..2290f49 100644 --- a/tip/Dockerfile +++ b/releaser/Dockerfile @@ -1,4 +1,4 @@ FROM python:3.9-slim-bullseye -COPY tip.py /tip.py +COPY releaser.py /releaser.py RUN pip install PyGithub --progress-bar off -CMD ["/tip.py"] +CMD ["/releaser.py"] diff --git a/tip/README.md b/releaser/README.md similarity index 84% rename from tip/README.md rename to releaser/README.md index d25639b..09b89be 100644 --- a/tip/README.md +++ b/releaser/README.md @@ -1,12 +1,12 @@ -# Tip +# Releaser -**Tip** is a Docker GitHub Action written in Python. +**Releaser** is a Docker GitHub Action written in Python. -**Tip** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds. -Combined with a workflow that is executed periodically, **Tip** allows to provide a fixed release name for users willing +**Releaser** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds. +Combined with a workflow that is executed periodically, **Releaser** allows to provide a fixed release name for users willing to use daily/nightly artifacts of a project. -Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Tip** can create a release and +Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Releaser** can create a release and upload assets. ## Context @@ -30,9 +30,9 @@ From a practical point of view, [actions/github-script](https://github.com/actio Still, it requires writing plain JavaScript. Alternatively, there are non-official GitHub API libraries available in other languages (see [docs.github.com: rest/overview/libraries](https://docs.github.com/en/rest/overview/libraries)). -**Tip** is based on [PyGithub/PyGithub](https://github.com/PyGithub/PyGithub), a Python client for the GitHub API. +**Releaser** is based on [PyGithub/PyGithub](https://github.com/PyGithub/PyGithub), a Python client for the GitHub API. -**Tip** was originally created in [eine/tip](https://github.com/eine/tip), as an enhanced alternative to using +**Releaser** was originally created in [eine/tip](https://github.com/eine/tip), as an enhanced alternative to using `actions/create-release` and `actions/upload-release-asset`, in order to cover certain use cases that were being migrated from Travis CI to GitHub Actions. The main limitation of GitHub's Actions was/is verbosity and not being possible to dynamically define the list of assets @@ -42,7 +42,7 @@ On the other hand, GitHub Actions artifacts do require login in order to downloa Conversely, assets of GitHub Releases can be downloaded without login. Therefore, in order to make CI results available to the widest audience, some projects prefer having tarballs available as assets. -In this context, one of the main use cases of **Tip** is pushing artifacts as release assets. +In this context, one of the main use cases of **Releaser** is pushing artifacts as release assets. Thus, the name of the Action. ## Usage @@ -72,7 +72,7 @@ jobs: # Update tag and pre-release # - Update (force-push) tag to the commit that is used in the workflow. # - Upload artifacts defined by the user. - - uses: pyTooling/Actions/tip@main + - uses: pyTooling/Actions/releaser@main with: token: ${{ secrets.GITHUB_TOKEN }} files: | @@ -82,11 +82,11 @@ jobs: ### Composite Action -The default implementation of **Tip** is a Container Action. +The default implementation of **Releaser** is a Container Action. Therefore, in each run, the container image is built before starting the job. -Alternatively, a Composite Action version is available: `uses: pyTooling/Actions/tip/composite@main`. +Alternatively, a Composite Action version is available: `uses: pyTooling/Actions/releaser/composite@main`. The Composite version installs the dependencies on the host (the runner environment), instead of using a container. -Both implementations are functionally equivalent from **Tip**'s point of view; however, the Composite Action allows users +Both implementations are functionally equivalent from **Releaser**'s point of view; however, the Composite Action allows users to tweak the version of Python by using [actions/setup-python](https://github.com/actions/setup-python) before. ## Options @@ -117,10 +117,10 @@ Whether to create releases from any tag or to treat some as snapshots. By defaul ## Advanced/complex use cases -**Tip** is essentially a very fine wrapper to use the GitHub Actions context data along with the classes +**Releaser** is essentially a very fine wrapper to use the GitHub Actions context data along with the classes and methods of PyGithub. -Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements might find it desirable to write their own Python script, instead of using **Tip**. +Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements might find it desirable to write their own Python script, instead of using **Releaser**. In fact, since `shell: python` is supported in GitHub Actions, using Python does *not* require any Action. For prototyping purposes, the following job might be useful: diff --git a/tip/action.yml b/releaser/action.yml similarity index 88% rename from tip/action.yml rename to releaser/action.yml index 07729a0..ee7e73a 100644 --- a/tip/action.yml +++ b/releaser/action.yml @@ -1,5 +1,5 @@ -name: 'tip' -description: "keep a pre-release always up-to-date" +name: 'Releaser' +description: 'Publish releases, upload assets and update tip/nightly tags' inputs: token: description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}' diff --git a/tip/composite/action.yml b/releaser/composite/action.yml similarity index 87% rename from tip/composite/action.yml rename to releaser/composite/action.yml index 255b4f6..4fde76f 100644 --- a/tip/composite/action.yml +++ b/releaser/composite/action.yml @@ -1,5 +1,5 @@ -name: 'tip' -description: "keep a pre-release always up-to-date" +name: 'Releaser' +description: 'Publish releases, upload assets and update tip/nightly tags' inputs: token: description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}' @@ -27,7 +27,7 @@ runs: run: pip install PyGithub --progress-bar off - shell: bash - run: ${{ github.action_path }}/../tip.py + run: ${{ github.action_path }}/../releaser.py env: INPUT_TOKEN: ${{ inputs.token }} INPUT_FILES: ${{ inputs.files }} diff --git a/tip/pyproject.toml b/releaser/pyproject.toml similarity index 100% rename from tip/pyproject.toml rename to releaser/pyproject.toml diff --git a/tip/tip.py b/releaser/releaser.py similarity index 100% rename from tip/tip.py rename to releaser/releaser.py From 5de7bff8b7d9f53a009c0151034ffe8a77d32a78 Mon Sep 17 00:00:00 2001 From: umarcor Date: Thu, 2 Dec 2021 16:06:26 +0100 Subject: [PATCH 37/39] Releaser/README: add subsection 'Troubleshooting' --- releaser/README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/releaser/README.md b/releaser/README.md index 09b89be..703e65f 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -80,6 +80,40 @@ jobs: README.md ``` +### Troubleshooting + +GitHub's internal connections seem not to be very stable; as a result, uploading artifacts as assets does produce +failures rather frequently, particularly if large tarballs are to be published. +When failures are produced, some assets are left in a broken state within the release. +**Releaser** tries to handle those cases by first uploading assets with a `tmp.*` name and then renaming them; if an existing +`tmp.*` is found, it is removed and the upload is retried. +Therefore, restarting the **Releaser** job should suffice for "fixing" a failing run. + +Note: + Currently, GitHub Actions does not allow restarting a single job. + That is unfortunate, because **Releaser** is typically used as the last dependent job in the workflows. + Hence, running **Releaser** again requires restarting the whole workflow. + Fortunately, restarting individual jobs is expected to be supported on GitHub Actions in the future. + See [github/roadmap#271](https://github.com/github/roadmap/issues/271) and [actions/runner#432](https://github.com/actions/runner/issues/432). + +If the tip/nightly release generated with **Releaser** is broken, and restarting the run cannot fix it, the recommended +procedure is the following: + +1. Go to `https://github.com///releases/edit/`. +2. Edit the assets to: + - Remove the ones with a warning symbol and/or named starting with `tmp.*`. + - Or, remove all of them. +3. Save the changes (click the `Update release` button) and restart the **Releaser** job in CI. +5. If that does still not work, remove the release and restart the **Releaser** job in CI. + +See also [eine/tip#160](https://github.com/eine/tip/issues/160). + +Note: + If all the assets are removed, or if the release itself is removed, tip/nightly assets won't be available for + users until the workflow is successfully run. + For instance, Action [setup-ghdl-ci](https://github.com/ghdl/setup-ghdl-ci) uses assets from [ghdl/ghdl: releases/tag/nightly](https://github.com/ghdl/ghdl/releases/tag/nightly). + Hence, it is recommended to try removing the conflictive assets only, in order to maximise the availability. + ### Composite Action The default implementation of **Releaser** is a Container Action. From da1aedeb6b7800dcabf77f303ae4bd6659158383 Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 7 Dec 2021 04:35:42 +0100 Subject: [PATCH 38/39] ci/TestReleaser: do not run on PRs --- .github/workflows/TestReleaser.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/TestReleaser.yml b/.github/workflows/TestReleaser.yml index 5ed94ac..b0c6937 100644 --- a/.github/workflows/TestReleaser.yml +++ b/.github/workflows/TestReleaser.yml @@ -7,7 +7,6 @@ on: - '!tip' branches: - '**' - pull_request: workflow_dispatch: schedule: - cron: '0 0 * * 4' From e0c0b37621c9b5b92dc699873e2d36715910f8aa Mon Sep 17 00:00:00 2001 From: umarcor Date: Tue, 7 Dec 2021 04:45:33 +0100 Subject: [PATCH 39/39] ci/TestReleaser: do not run on 'v*' tags or 'r*' branches --- .github/workflows/TestReleaser.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/TestReleaser.yml b/.github/workflows/TestReleaser.yml index b0c6937..e9035e4 100644 --- a/.github/workflows/TestReleaser.yml +++ b/.github/workflows/TestReleaser.yml @@ -5,8 +5,10 @@ on: tags: - '*' - '!tip' + - '!v*' branches: - '**' + - '!r*' workflow_dispatch: schedule: - cron: '0 0 * * 4'