Compare commits

..

1 Commits

Author SHA1 Message Date
Rik Huijzer
cbfbe4981e Add cache-name to main example
This patch suggests to add `cache-name` to the main example since most Julia projects test multiple versions in a matrix.
2024-05-21 15:23:32 +02:00
5 changed files with 52 additions and 159 deletions

View File

@@ -59,7 +59,7 @@ jobs:
env:
JULIA_DEPOT_PATH: /tmp/julia-depot
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
- name: Set cache-name
id: cache-name
shell: bash
@@ -117,7 +117,7 @@ jobs:
env:
JULIA_DEPOT_PATH: /tmp/julia-depot
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
- uses: julia-actions/setup-julia@v2
with:
version: ${{ matrix.version }}
@@ -161,7 +161,7 @@ jobs:
outputs:
cache-name: ${{ steps.cache-name.outputs.cache-name }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
- name: Set cache-name
id: cache-name
run: |
@@ -193,7 +193,7 @@ jobs:
needs: test-save-nomatrix
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
- name: Restore cache
id: cache
uses: ./
@@ -233,7 +233,7 @@ jobs:
outputs:
cache-name: ${{ steps.cache-name.outputs.cache-name }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
- name: Set cache-name
id: cache-name
run: |
@@ -259,7 +259,7 @@ jobs:
needs: test-save-cloned-registry
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
- name: Add General registry clone
shell: julia --color=yes {0}
run: |

View File

@@ -19,20 +19,29 @@ permissions:
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
version:
- '1.6'
- '1'
steps:
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
- uses: julia-actions/cache@v2
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
- uses: actions/checkout@v4
- uses: julia-actions/setup-julia@v2
- uses: julia-actions/cache@v2
with:
cache-name: 'test-${{ matrix.os }}'
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
```
By default all depot directories called out below are cached.
### Requirements
- `jq`: This action uses [`jq`](https://github.com/jqlang/jq) to parse JSON. For GitHub-hosted runners, `jq` is installed by default. On self-hosted runners and custom containers, if `jq` is not already available, this action will automatically use [`dcarbone/install-jq-action`](https://github.com/dcarbone/install-jq-action) to install `jq` (Note: `dcarbone/install-jq-action` requires that `curl` is installed; this may not always be the case in custom containers and self-hosted runners).
- `bash`: This action requires `bash`. For GitHub-hosted runners `bash` is installed by default. Self-hosted runners will need to ensure that `bash` is installed and available on the `PATH`.
This action uses [`jq`](https://github.com/jqlang/jq) to parse JSON.
`jq` is installed by default in GitHub-hosted runners.
[`dcarbone/install-jq-action`](https://github.com/dcarbone/install-jq-action) is used to check that `jq` is available and install it if not.
**Note:** installing `jq` with `dcarbone/install-jq-action` requires that curl is available; this may not be the case in custom containers.
### Optional Inputs
@@ -64,9 +73,6 @@ By default, this action removes caches that were previously made by jobs on the
GitHub automatically removes old caches after a certain period or when the repository cache allocation is full.
It is, however, more efficient to explicitly remove old caches to improve caching for less frequently run jobs.
For more information about Github caching generically, for example how to manually delete caches, see
[this Github documentation page](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#managing-caches).
### Cache keys
The cache key that the cache will be saved as is based on:
@@ -77,8 +83,8 @@ The cache key that the cache will be saved as is based on:
- The run attempt number
> [!NOTE]
> If there is job concurrency that is not fully defined by a matrix you should ensure that `cache-name` is
> unique for each concurrent job, otherwise caching may not be effective.
> If in your workflow if you do not use a matrix for concurrency you should make `cache-name` such that it is unique for
> concurrent jobs, otherwise caching may not be effective.
### Cache Retention

View File

@@ -47,18 +47,12 @@ outputs:
cache-hit:
description: A boolean value to indicate an exact match was found for the primary key. Returns "" when the key is new. Forwarded from actions/cache.
value: ${{ steps.hit.outputs.cache-hit }}
cache-paths:
description: The paths that were cached
value: ${{ steps.paths.outputs.cache-paths }}
cache-key:
description: The full cache key used
value: ${{ steps.keys.outputs.key }}
runs:
using: 'composite'
steps:
- name: Install jq
uses: dcarbone/install-jq-action@f0e10f46ff84f4d32178b4b76e1ef180b16f82c3 # v3.1.1
uses: dcarbone/install-jq-action@8867ddb4788346d7c22b72ea2e2ffe4d514c7bcb
with:
force: false # Skip install when an existing `jq` is present
@@ -72,12 +66,6 @@ runs:
else
depot="~/.julia"
fi
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then
depot="${depot/#\~/$USERPROFILE}" # Windows paths
depot="${depot//\\//}" # Replace backslashes with forward slashes
else
depot="${depot/#\~/$HOME}" # Unix-like paths
fi
echo "depot=$depot" | tee -a "$GITHUB_OUTPUT"
cache_paths=()
@@ -87,7 +75,7 @@ runs:
[ "${{ inputs.cache-packages }}" = "true" ] && cache_paths+=("$packages_path")
registries_path="${depot}/registries"
if [ "${{ inputs.cache-registries }}" = "true" ]; then
if [ ! -d "${registries_path}" ]; then
if [ ! -d "${registries_path/#\~/$HOME}" ]; then
cache_paths+=("$registries_path")
else
echo "::warning::Julia depot registries already exist. Skipping restoring of cached registries to avoid potential merge conflicts when updating. Please ensure that \`julia-actions/cache\` precedes any workflow steps which add registries."
@@ -127,7 +115,7 @@ runs:
env:
MATRIX_JSON: ${{ toJSON(matrix) }}
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
- uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
id: cache
with:
path: |
@@ -144,44 +132,12 @@ runs:
du -shc ${{ steps.paths.outputs.depot }}/* || true
shell: bash
# issue https://github.com/julia-actions/cache/issues/110
# Pkg may not run `Registry.update()` if a manifest exists, which may exist because of a
# `Pkg.dev` call or because one is added to the repo. So be safe and update cached registries here.
# Older (~v1.0) versions of julia that don't have `Pkg.Registry.update()` seem to always update registries in
# Pkg operations. So this is only necessary for newer julia versions.
- name: Update any cached registries
if: ${{ inputs.cache-registries == 'true' }}
continue-on-error: true
run: |
if [ -d "${{ steps.paths.outputs.depot }}/registries" ] && [ -n "$(ls -A "${{ steps.paths.outputs.depot }}/registries")" ]; then
echo "Registries directory exists and is non-empty. Updating any registries"
julia -e "import Pkg; isdefined(Pkg, :Registry) && Pkg.Registry.update();"
else
echo "Registries directory does not exist or is empty. Skipping registry update"
fi
shell: bash
# GitHub actions cache entries are immutable and cannot be updated. In order to have both the Julia
# depot cache be up-to-date and avoid storing redundant cache entries we'll manually cleanup old
# cache entries before the new cache is saved. However, we need to be careful with our manual
# cleanup as otherwise we can cause cache misses for jobs which would have normally had a cache hit.
# Some scenarios to keep in mind include:
#
# - Job failures result in the post-action for `actions/cache` being skipped. If we delete all cache
# entries for the branch we may have no cache entry available for the next run.
# - We should avoid deleting old cache entries for the default branch since these entries serve as
# the fallback if no earlier cache entry exists on a branch. We can rely on GitHub's default cache
# eviction policy here which will remove the oldest cache entry first.
#
# References:
# - https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
# - https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
# github and actions/cache doesn't provide a way to update a cache at a given key, so we delete any
# that match the restore key just before saving the new cache
# Not windows
- uses: pyTooling/Actions/with-post-step@42e17fae05f224e5ac3d79d021a4e3577878efe5 # v4.2.2
if: ${{ inputs.delete-old-caches != 'false' &&
github.ref != format('refs/heads/{0}', github.event.repository.default_branch) &&
runner.OS != 'Windows' }}
- uses: pyTooling/Actions/with-post-step@e9d0dc3dba9fda45f195946858708f60c0240caf # v1.0.5
if: ${{ inputs.delete-old-caches != 'false' && runner.OS != 'Windows' }}
with:
# seems like there has to be a `main` step in this action. Could list caches for info if we wanted
# main: julia ${{ github.action_path }}/handle_caches.jl "${{ github.repository }}" "list"
@@ -191,10 +147,8 @@ runs:
GH_TOKEN: ${{ inputs.token }}
# Windows (because this action uses command prompt on windows)
- uses: pyTooling/Actions/with-post-step@42e17fae05f224e5ac3d79d021a4e3577878efe5 # v4.2.2
if: ${{ inputs.delete-old-caches != 'false' &&
github.ref != format('refs/heads/{0}', github.event.repository.default_branch) &&
runner.OS == 'Windows' }}
- uses: pyTooling/Actions/with-post-step@e9d0dc3dba9fda45f195946858708f60c0240caf # v1.0.5
if: ${{ inputs.delete-old-caches != 'false' && runner.OS == 'Windows' }}
with:
main: echo ""
post: cd %GITHUB_ACTION_PATH% && julia handle_caches.jl rm "${{ github.repository }}" "${{ steps.keys.outputs.restore-key }}" "${{ github.ref }}" "${{ inputs.delete-old-caches != 'required' }}"

View File

@@ -1,56 +0,0 @@
# Making a new release
In this guide, as an example, `v2.2.0` refers to the version number of the new release that you want to make.
## Part 1: Use the Git CLI to create and push the Git tags
Step 1: Create a new lightweight tag of the form `vMAJOR.MINOR.PATCH`.
```bash
git clone git@github.com:julia-actions/cache.git
cd cache
git fetch --all --tags
git checkout main
git --no-pager log -1
# Take note of the commit hash here.
# Now, create a new lightweight tag of the form `vMAJOR.MINOR.PATCH`.
#
# Replace `commit_hash` with the commit hash that you obtained from the
# `git log -1` step.
#
# Replace `v2.2.0` with the actual version number that you want to use.
git tag v2.2.0 commit_hash
```
Step 2: Once you've created the new release, you need to update the `v2` tag to point to the new release. For example, suppose that the previous release was `v2.1.0`, and suppose that you just created the new release `v2.2.0`. You need to update the `v2` tag so that it points to `v2.2.0`. Here are the commands:
```bash
# Create/update the new v2 tag locally, where the new v2 tag will point to the
# release that you created in the previous step.
#
# Make sure to change `v2.2.0` to the actual value for the tag that you just
# created in the previous step.
#
# The `-f` flag forcibly overwrites the old
# `v2` tag (if it exists).
git tag -f v2 v2.2.0
```
Step 3: Now you need to push the tags:
```bash
# Regular-push the new `v2.2.0` tag:
git push origin tag v2.2.0
# Force-push the new v2 tag:
git push origin tag v2 --force
```
## Part 2: Create the GitHub Release
Go to the [Releases](https://github.com/julia-actions/cache/releases) section of this repo and create a new release (using the GitHub web interface).
For the "choose a tag" drop-down field, select the `v2.2.0` tag that you created and pushed in Part 1 of this guide.

View File

@@ -10,48 +10,37 @@ function handle_caches()
repo, restore_key, ref = ARGS[2:4]
allow_failure = ARGS[5] == "true"
endpoint = "/repos/$repo/actions/caches"
page = 1
per_page = 100
skipped = String[]
deleted = String[]
failed = String[]
escaped_restore_key = replace(restore_key, "\"" => "\\\"")
query = ".actions_caches[] | select(.key | startswith(\"$escaped_restore_key\")) | .id"
deletions = String[]
failures = String[]
while 1 <= page <= 5 # limit to avoid accidental rate limiting
# https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#list-github-actions-caches-for-a-repository
# Note: The `key` field matches on the full key or a prefix.
cmd = ```
gh api -X GET /repos/$repo/actions/caches
--field per_page=$per_page
--field page=$page
--field ref=$ref
--field key=$restore_key
--field sort=last_accessed_at
--field direction=desc
--jq '.actions_caches[].id'
```
cmd = `gh api -X GET $endpoint -F ref=$ref -F per_page=$per_page -F page=$page --jq $query`
ids = split(read(cmd, String); keepempty=false)
page = length(ids) == per_page ? page + 1 : -1
# Avoid deleting the latest used cache entry. This is particularly important for
# job failures where a new cache entry will not be saved after this.
page == 1 && !isempty(ids) && push!(skipped, popfirst!(ids))
# We can delete all cache entries on this branch that matches the restore key
# because the new cache is saved later.
for id in ids
try
run(`gh cache delete $id --repo $repo`)
push!(deleted, id)
push!(deletions, id)
catch e
@error e
push!(failed, id)
push!(failures, id)
end
end
page = length(ids) == per_page ? page + 1 : -1
end
if isempty(skipped) && isempty(deleted) && isempty(failed)
if isempty(failures) && isempty(deletions)
println("No existing caches found on ref `$ref` matching restore key `$restore_key`")
else
if !isempty(failed)
println("Failed to delete $(length(failed)) existing caches on ref `$ref` matching restore key `$restore_key`")
println.(failed)
if !isempty(failures)
println("Failed to delete $(length(failures)) existing caches on ref `$ref` matching restore key `$restore_key`")
println.(failures)
@info """
To delete caches you need to grant the following to the default `GITHUB_TOKEN` by adding
this to your workflow:
@@ -66,9 +55,9 @@ function handle_caches()
"""
allow_failure || exit(1)
end
if !isempty(deleted)
println("Deleted $(length(deleted)) caches on ref `$ref` matching restore key `$restore_key`")
println.(deleted)
if !isempty(deletions)
println("Deleted $(length(deletions)) caches on ref `$ref` matching restore key `$restore_key`")
println.(deletions)
end
end
else