Tip: address codacy warnings

add DEVELOPMENT.md

.github/workflows/Tip.yml

@@ -0,0 +1,132 @@
+name: Tip
+
+on:
+  push:
+    tags:
+      - '*'
+      - '!tip'
+    branches:
+      - '**'
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 4'
+
+env:
+  CI: true
+
+jobs:
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt
+
+    - name: Single
+      uses: ./tip
+      with:
+        rm: true
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifact-*.txt
+
+    - name: List
+      uses: ./tip
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: |
+          artifact-*.txt
+          README.md
+
+    - name: Add artifacts/*.txt
+      run: |
+        mkdir artifacts
+        echo "Build some tool and generate some artifacts" > artifacts/artifact.txt
+        touch artifacts/empty_file.txt
+
+    - name: Single in subdir
+      uses: ./tip
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifacts/artifact.txt
+
+    - name: Add artifacts/*.md
+      run: |
+        echo "tip hello" > artifacts/hello.md
+        echo "tip world" > artifacts/world.md
+
+    - name: Directory wildcard
+      uses: ./tip
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifacts/*
+
+    - name: Add artifacts/subdir
+      run: |
+        mkdir artifacts/subdir
+        echo "Test recursive glob" > artifacts/subdir/deep_file.txt
+
+    - name: Directory wildcard (recursive)
+      uses: ./tip
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifacts/**
+
+
+  test-composite:
+    needs: test
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - run: echo "Build some tool and generate some (versioned) artifacts" > artifact-$(date -u +"%Y-%m-%dT%H-%M-%SZ").txt
+
+    - name: Single
+      uses: ./tip/composite
+      with:
+        rm: true
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifact-*.txt
+
+    - name: List
+      uses: ./tip/composite
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: |
+          artifact-*.txt
+          README.md
+
+    - name: Add artifacts/*.txt
+      run: |
+        mkdir artifacts
+        echo "Build some tool and generate some artifacts" > artifacts/artifact.txt
+        touch artifacts/empty_file.txt
+
+    - name: Single in subdir
+      uses: ./tip/composite
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifacts/artifact.txt
+
+    - name: Add artifacts/*.md
+      run: |
+        echo "tip hello" > artifacts/hello.md
+        echo "tip world" > artifacts/world.md
+
+    - name: Directory wildcard
+      uses: ./tip/composite
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifacts/*
+
+    - name: Add artifacts/subdir
+      run: |
+        mkdir artifacts/subdir
+        echo "Test recursive glob" > artifacts/subdir/deep_file.txt
+
+    - name: Directory wildcard (recursive)
+      uses: ./tip/composite
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: artifacts/**

.github/workflows/UnitTesting.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
     inputs:
       jobs:
-        description: 'Space separated list of Python versions to run tests with.'
+        description: 'JSON list with field <python>, telling the versions to run tests with.'
         required: true
         type: string
       requirements:

DEVELOPMENT.md

@@ -0,0 +1,22 @@
+# Development
+
+## Tagging/versioning
+
+See context in [#5](https://github.com/pyTooling/Actions/issues/5).
+
+Tag new releases in the `main` branch using a semver compatible value, starting with `v`:
+
+```sh
+git checkout main
+git tag v0.0.0
+git push upstream v0.0.0
+```
+
+Move the corresponding release branch (starting with `r`) forward by creating a merge commit, and using the merged tag
+as the commit message:
+
+```sh
+git checkout r0
+git merge --no-ff -m 'v0.0.0' v0.0.0
+git push upstream r0
+```

ExamplePipeline.yml

@@ -1,4 +1,4 @@
-name: Unit Testing, Coverage Collection, Package, Release, Documentation and Publish
+name: Pipeline
 
 on:
   workflow_dispatch:
@@ -49,17 +49,8 @@ jobs:
       requirements: '-r tests/requirements.txt'
       report: 'htmlmypy'
 
-  Release:
-    uses: pyTooling/Actions/.github/workflows/Release.yml@main
-    if: startsWith(github.ref, 'refs/tags')
-    needs:
-      - UnitTesting
-      - Coverage
-      - StaticTypeCheck
-
   Package:
     uses: pyTooling/Actions/.github/workflows/Package.yml@main
-    if: startsWith(github.ref, 'refs/tags')
     needs:
       - Params
       - Coverage
@@ -69,6 +60,15 @@ jobs:
       python_version: ${{ fromJson(needs.Params.outputs.params).python_version }}
       requirements: 'wheel'
 
+  Release:
+    uses: pyTooling/Actions/.github/workflows/Release.yml@main
+    if: startsWith(github.ref, 'refs/tags')
+    needs:
+      - UnitTesting
+      - Coverage
+      - StaticTypeCheck
+      - Package
+
   PublishOnPyPI:
     uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@main
     if: startsWith(github.ref, 'refs/tags')
@@ -117,6 +117,7 @@ jobs:
     uses: pyTooling/Actions/.github/workflows/ArtifactCleanUp.yml@main
     needs:
       - Params
+      - UnitTesting
       - Coverage
       - StaticTypeCheck
       - BuildTheDocs

tip/Dockerfile

@@ -0,0 +1,4 @@
+FROM python:3.9-slim-bullseye
+COPY tip.py /tip.py
+RUN pip install PyGithub --progress-bar off
+ENTRYPOINT ["/tip.py"]

tip/README.md

@@ -0,0 +1,152 @@
+# Tip
+
+**Tip** is a Docker GitHub Action written in Python.
+
+**Tip** allows to keep a GitHub Release of type pre-release and its artifacts up to date with latest builds.
+Combined with a workflow that is executed periodically, **Tip** allows to provide a fixed release name for users willing
+to use daily/nightly artifacts of a project.
+
+Furthermore, when any [semver](https://semver.org) compilant tagged commit is pushed, **Tip** can create a release and
+upload assets.
+
+## Context
+
+GitHub provides official clients for the GitHub API through [github.com/octokit](https://github.com/octokit):
+
+- [octokit.js](https://github.com/octokit/octokit.js) ([octokit.github.io/rest.js](https://octokit.github.io/rest.js))
+- [octokit.rb](https://github.com/octokit/octokit.rb) ([octokit.github.io/octokit.rb](http://octokit.github.io/octokit.rb))
+- [octokit.net](https://github.com/octokit/octokit.net) ([octokitnet.rtfd.io](https://octokitnet.rtfd.io))
+
+When GitHub Actions was released in 2019, two Actions were made available through [github.com/actions](https://github.com/actions) for dealing with GitHub Releases:
+
+- [actions/create-release](https://github.com/actions/create-release)
+- [actions/upload-release-asset](https://github.com/actions/upload-release-asset)
+
+However, those Actions were contributed by an employee in spare time, not officially supported by GitHub.
+Therefore, they were unmaintained before GitHub Actions was out of the private beta (see [actions/upload-release-asset#58](https://github.com/actions/upload-release-asset/issues/58)) and, a year later, archived.
+Those Actions are based on [actions/toolkit](https://github.com/actions/toolkit)'s hydrated version of octokit.js.
+
+From a practical point of view, [actions/github-script](https://github.com/actions/github-script) is the natural replacement to those Actions, since it allows to use a pre-authenticated octokit.js client along with the workflow run context.
+Still, it requires writing plain JavaScript.
+
+Alternatively, there are non-official GitHub API libraries available in other languages (see [docs.github.com: rest/overview/libraries](https://docs.github.com/en/rest/overview/libraries)).
+**Tip** is based on [PyGithub/PyGithub](https://github.com/PyGithub/PyGithub), a Python client for the GitHub API.
+
+**Tip** was originally created in [eine/tip](https://github.com/eine/tip), as an enhanced alternative to using
+`actions/create-release` and `actions/upload-release-asset`, in order to cover certain use cases that were being
+migrated from Travis CI to GitHub Actions.
+The main limitation of GitHub's Actions was/is verbosity and not being possible to dynamically define the list of assets
+to be uploaded.
+
+On the other hand, GitHub Actions artifacts do require login in order to download them.
+Conversely, assets of GitHub Releases can be downloaded without login.
+Therefore, in order to make CI results available to the widest audience, some projects prefer having tarballs available
+as assets.
+In this context, one of the main use cases of **Tip** is pushing artifacts as release assets.
+Thus, the name of the Action.
+
+## Usage
+
+The following block shows a minimal YAML workflow file:
+
+```yml
+name: 'workflow'
+
+on:
+  schedule:
+    - cron: '0 0 * * 5'
+
+jobs:
+  mwe:
+    runs-on: ubuntu-latest
+    steps:
+
+    # Clone repository
+    - uses: actions/checkout@v2
+
+    # Build your application, tool, artifacts, etc.
+    - name: Build
+      run: |
+        echo "Build some tool and generate some artifacts" > artifact.txt
+
+    # Update tag and pre-release
+    # - Update (force-push) tag to the commit that is used in the workflow.
+    # - Upload artifacts defined by the user.
+    - uses: pyTooling/Actions/tip@main
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        files: |
+          artifact.txt
+          README.md
+```
+
+### Composite Action
+
+The default implementation of **Tip** is a Container Action.
+Therefore, in each run, the container image is built before starting the job.
+Alternatively, a Composite Action version is available: `uses: pyTooling/Actions/tip/composite@main`.
+The Composite version installs the dependencies on the host (the runner environment), instead of using a container.
+Both implementations are functionally equivalent from **Tip**'s point of view; however, the Composite Action allows users
+to tweak the version of Python by using [actions/setup-python](https://github.com/actions/setup-python) before.
+
+## Options
+
+All options can be optionally provided as environment variables: `INPUT_TOKEN`, `INPUT_FILES`, `INPUT_TAG`, `INPUT_RM` and/or `INPUT_SNAPSHOTS`.
+
+### token (required)
+
+Token to make authenticated API calls; can be passed in using `{{ secrets.GITHUB_TOKEN }}`.
+
+### files (required)
+
+Either a single filename/pattern or a multi-line list can be provided. All the artifacts are uploaded regardless of the hierarchy.
+
+For creating/updating a release without uploading assets, set `files: none`.
+
+### tag
+
+The default tag name for the tip/nightly pre-release is `tip`, but it can be optionally overriden through option `tag`.
+
+### rm
+
+Set option `rm` to `true` for systematically removing previous artifacts (e.g. old versions). Otherwise (by default), all previours artifacts are preserved or overwritten.
+
+### snapshots
+
+Whether to create releases from any tag or to treat some as snapshots. By default, all the tags with non-empty `prerelease` field (see [semver.org: Is there a suggested regular expression (RegEx) to check a SemVer string?](https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string)) are considered snapshots; neither a release is created nor assets are uploaded.
+
+## Advanced/complex use cases
+
+**Tip** is essentially a very fine wrapper to use the GitHub Actions context data along with the classes
+and methods of PyGithub.
+
+Similarly to [actions/github-script](https://github.com/actions/github-script), users with advanced/complex requirements might find it desirable to write their own Python script, instead of using **Tip**.
+In fact, since `shell: python` is supported in GitHub Actions, using Python does *not* require any Action.
+For prototyping purposes, the following job might be useful:
+
+```yml
+  Release:
+    name: '📦 Release'
+    runs-on: ubuntu-latest
+    needs:
+      - ...
+    if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/master' || contains(github.ref, 'refs/tags/'))
+    steps:
+
+      - uses: actions/download-artifact@v2
+
+      - shell: bash
+        run: pip install PyGithub --progress-bar off
+
+      - name: Set list of files for uploading
+        id: files
+        shell: python
+        run: |
+          from github import Github
+          print("· Get GitHub API handler (authenticate)")
+          gh = Github('${{ github.token }}')
+          print("· Get Repository handler")
+          gh_repo = gh.get_repo('${{ github.repository }}')
+```
+
+Find a non-trivial use case at [msys2/msys2-autobuild](https://github.com/msys2/msys2-autobuild).

tip/action.yml

@@ -0,0 +1,24 @@
+name: 'tip'
+description: "keep a pre-release always up-to-date"
+inputs:
+  token:
+    description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}'
+    required: true
+  files:
+    description: 'Multi-line list of glob patterns describing the artifacts to be uploaded'
+    required: true
+  tag:
+    description: 'Name of the tag that corresponds to the tip/nightly pre-release'
+    required: false
+    default: tip
+  rm:
+    description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name'
+    required: false
+    default: false
+  snapshots:
+    description: 'Whether to create releases from any tag or to treat some as snapshots'
+    required: false
+    default: true
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

tip/composite/action.yml

@@ -0,0 +1,36 @@
+name: 'tip'
+description: "keep a pre-release always up-to-date"
+inputs:
+  token:
+    description: 'Token to make authenticated API calls; can be passed in using {{ secrets.GITHUB_TOKEN }}'
+    required: true
+  files:
+    description: 'Multi-line list of glob patterns describing the artifacts to be uploaded'
+    required: true
+  tag:
+    description: 'Name of the tag that corresponds to the tip/nightly pre-release'
+    required: false
+    default: tip
+  rm:
+    description: 'Whether to delete all the previous artifacts, or only replacing the ones with the same name'
+    required: false
+    default: false
+  snapshots:
+    description: 'Whether to create releases from any tag or to treat some as snapshots'
+    required: false
+    default: true
+runs:
+  using: 'composite'
+  steps:
+
+  - shell: bash
+    run: pip install PyGithub --progress-bar off
+
+  - shell: bash
+    run: ${{ github.action_path }}/../tip.py
+    env:
+      INPUT_TOKEN: ${{ inputs.token }}
+      INPUT_FILES: ${{ inputs.files }}
+      INPUT_TAG: ${{ inputs.tag }}
+      INPUT_RM: ${{ inputs.rm }}
+      INPUT_SNAPSHOTS: ${{ inputs.snapshots }}

tip/pyproject.toml

@@ -0,0 +1,2 @@
+[tool.black]
+line-length = 120

tip/tip.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+
+import re
+from sys import argv, stdout, exit as sys_exit
+from os import environ, getenv
+from glob import glob
+from pathlib import Path
+from github import Github
+from github import GithubException
+
+print("· Get list of artifacts to be uploaded")
+
+args = []
+files = []
+
+if "INPUT_FILES" in environ:
+    args = environ["INPUT_FILES"].split()
+
+if len(argv) > 1:
+    args = args + argv[1:]
+
+if len(args) == 1 and args[0] == "none":
+    files = []
+    print("! Skipping 'files' because it's set to 'none")
+else:
+    if len(args) == 0:
+        stdout.flush()
+        raise (Exception("Glob patterns need to be provided as positional arguments or through envvar 'INPUT_FILES'!"))
+
+    for item in args:
+        items = [fname for fname in glob(item, recursive=True) if not Path(fname).is_dir()]
+        print(f"  glob({item!s}):")
+        for fname in items:
+            if Path(fname).stat().st_size == 0:
+                print(f"  - ! Skipping empty file {fname!s}")
+                continue
+            print(f"  - {fname!s}")
+            files += [fname]
+
+    if len(files) < 1:
+        stdout.flush()
+        raise (Exception("Empty list of files to upload/update!"))
+
+print("· Get GitHub API handler (authenticate)")
+
+if "GITHUB_TOKEN" in environ:
+    gh = Github(environ["GITHUB_TOKEN"])
+elif "INPUT_TOKEN" in environ:
+    gh = Github(environ["INPUT_TOKEN"])
+else:
+    if "GITHUB_USER" not in environ or "GITHUB_PASS" not in environ:
+        stdout.flush()
+        raise (
+            Exception(
+                "Need credentials to authenticate! Please, provide 'GITHUB_TOKEN', 'INPUT_TOKEN', or 'GITHUB_USER' and 'GITHUB_PASS'"
+            )
+        )
+    gh = Github(environ["GITHUB_USER"], environ["GITHUB_PASS"])
+
+print("· Get Repository handler")
+
+if "GITHUB_REPOSITORY" not in environ:
+    stdout.flush()
+    raise (Exception("Repository name not defined! Please set 'GITHUB_REPOSITORY"))
+
+gh_repo = gh.get_repo(environ["GITHUB_REPOSITORY"])
+
+print("· Get Release handler")
+
+tag = getenv("INPUT_TAG", "tip")
+
+env_tag = None
+gh_ref = environ["GITHUB_REF"]
+is_prerelease = True
+is_draft = False
+
+if gh_ref[0:10] == "refs/tags/":
+    env_tag = gh_ref[10:]
+    if env_tag != tag:
+        rexp = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+        semver = re.search(rexp, env_tag)
+        if semver == None and env_tag[0] == "v":
+            semver = re.search(rexp, env_tag[1:])
+        tag = env_tag
+        if semver == None:
+            print(f"! Could not get semver from {gh_ref!s}")
+            print(f"! Treat tag '{tag!s}' as a release")
+            is_prerelease = False
+        else:
+            if semver.group("prerelease") is None:
+                # is a regular semver compilant tag
+                is_prerelease = False
+            elif getenv("INPUT_SNAPSHOTS", "true") == "true":
+                # is semver compilant prerelease tag, thus a snapshot (we skip it)
+                print("! Skipping snapshot prerelease")
+                sys_exit()
+
+gh_tag = None
+try:
+    gh_tag = gh_repo.get_git_ref(f"tags/{tag!s}")
+except Exception as e:
+    stdout.flush()
+
+if gh_tag:
+    try:
+        gh_release = gh_repo.get_release(tag)
+    except Exception as e:
+        gh_release = gh_repo.create_git_release(tag, tag, "", draft=True, prerelease=is_prerelease)
+        is_draft = True
+else:
+    err_msg = f"Tag/release '{tag!s}' does not exist and could not create it!"
+    if "GITHUB_SHA" not in environ:
+        raise (Exception(err_msg))
+    try:
+        gh_release = gh_repo.create_git_tag_and_release(
+            tag, "", tag, "", environ["GITHUB_SHA"], "commit", draft=True, prerelease=is_prerelease
+        )
+        is_draft = True
+    except Exception as e:
+        raise (Exception(err_msg))
+
+print("· Cleanup and/or upload artifacts")
+
+artifacts = files
+
+assets = gh_release.get_assets()
+
+
+def delete_asset_by_name(name):
+    for asset in assets:
+        if asset.name == name:
+            asset.delete_asset()
+            return
+
+
+def upload_asset(artifact, name):
+    try:
+        return gh_release.upload_asset(artifact, name=name)
+    except GithubException as ex:
+        if "already_exists" in [err["code"] for err in ex.data["errors"]]:
+            print(f"   - {name} exists already! deleting...")
+            delete_asset_by_name(name)
+        else:
+            print(f"   - uploading failed: {ex}")
+    except Exception as ex:
+        print(f"   - uploading failed: {ex}")
+
+    print(f"   - retry uploading {name}...")
+    return gh_release.upload_asset(artifact, name=name)
+
+
+def replace_asset(artifacts, asset):
+    print(f" > {asset!s}\n   {asset.name!s}:")
+    for artifact in artifacts:
+        aname = str(Path(artifact).name)
+        if asset.name == aname:
+            print(f"   - uploading tmp.{aname!s}...")
+            new_asset = upload_asset(artifact, name=f"tmp.{aname!s}")
+            print(f"   - removing...{aname!s}")
+            asset.delete_asset()
+            print(f"   - renaming tmp.{aname!s} to {aname!s}...")
+            new_asset.update_asset(aname, label=aname)
+            artifacts.remove(artifact)
+            return
+    print("   - keep")
+
+
+if getenv("INPUT_RM", "false") == "true":
+    print("· RM set. All previous assets are being cleared...")
+    for asset in assets:
+        print(f" - {asset.name}")
+        asset.delete_asset()
+else:
+    for asset in assets:
+        replace_asset(artifacts, asset)
+
+for artifact in artifacts:
+    print(f" > {artifact!s}:\n   - uploading...")
+    gh_release.upload_asset(artifact)
+
+stdout.flush()
+print("· Update Release reference (force-push tag)")
+
+if is_draft:
+    # Unfortunately, it seems not possible to update fields 'created_at' or 'published_at'.
+    print(" > Update (pre-)release")
+    gh_release.update_release(
+        gh_release.title,
+        "" if gh_release.body is None else gh_release.body,
+        draft=False,
+        prerelease=is_prerelease,
+        tag_name=gh_release.tag_name,
+        target_commitish=gh_release.target_commitish,
+    )
+
+if ("GITHUB_SHA" in environ) and (env_tag is None):
+    sha = environ["GITHUB_SHA"]
+    print(f" > Force-push '{tag!s}' to {sha!s}")
+    gh_repo.get_git_ref(f"tags/{tag!s}").edit(sha)