mirror of
https://github.com/julia-actions/cache.git
synced 2026-02-12 09:26:53 +08:00
Unique cache-key for job matrix objects (#88)
* Unique cache-key for job matrix objects * Update workflow to use object in job matrix * Restore key should always match startswith * fixup! Unique cache-key for job matrix objects * Debug no-matrix * Tests require overriding workflow/job * Skip generating matrix_key when no matrix is used * Install jq for self-hosted runners * fixup! Install jq for self-hosted runners * Skip install when not needed * fixup! Skip install when not needed * fixup! Skip install when not needed * Improve `cache-name` description Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com> * Update description in README * Use actions/checkout@v4 in example * add missing period --------- Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>
This commit is contained in:
Curtis Vogt
18
.github/workflows/CI.yml
vendored
18
.github/workflows/CI.yml
vendored
@@ -36,7 +36,13 @@ jobs:
|
||||
needs: generate-key
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macOS-latest]
|
||||
dep:
|
||||
- name: pandoc_jll
|
||||
version: "3"
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- windows-latest
|
||||
- macOS-latest
|
||||
fail-fast: false
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
@@ -55,7 +61,7 @@ jobs:
|
||||
@assert !isdir(dir)
|
||||
- name: Install a small binary
|
||||
shell: 'julia --color=yes {0}'
|
||||
run: 'using Pkg; Pkg.add("pandoc_jll")'
|
||||
run: 'using Pkg; Pkg.add(PackageSpec(name="${{ matrix.dep.name }}", version="${{ matrix.dep.version }}"))'
|
||||
|
||||
# Do tests with no matrix also given the matrix is auto-included in cache key
|
||||
test-save-nomatrix:
|
||||
@@ -81,7 +87,13 @@ jobs:
|
||||
needs: [generate-key, test-save]
|
||||
strategy:
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macOS-latest]
|
||||
dep:
|
||||
- name: pandoc_jll
|
||||
version: "3"
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- windows-latest
|
||||
- macOS-latest
|
||||
fail-fast: false
|
||||
runs-on: ${{ matrix.os }}
|
||||
env:
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v4
|
||||
- uses: julia-actions/setup-julia@v1
|
||||
- uses: julia-actions/cache@v1
|
||||
- uses: julia-actions/julia-buildpkg@v1
|
||||
@@ -31,7 +31,7 @@ By default all depot directories called out below are cached.
|
||||
|
||||
### Optional Inputs
|
||||
|
||||
- `cache-name` - The cache key prefix. Defaults to `julia-cache-${{ github.workflow }}-${{ github.job }}`. The key body automatically includes matrix vars and the OS. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
|
||||
- `cache-name` - The cache key prefix. Defaults to `julia-cache;workflow=${{ github.workflow }};job=${{ github.job }}`. The key body automatically includes the OS and, unless disabled with `include-matrix`, the matrix vars. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
|
||||
- `include-matrix` - Whether to include the matrix values when constructing the cache key. Defaults to `true`.
|
||||
- `depot` - Path to a Julia [depot](https://pkgdocs.julialang.org/v1/glossary/) directory where cached data will be saved to and restored from. Defaults to the first depot in [`JULIA_DEPOT_PATH`](https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH) if specified. Otherwise, defaults to `~/.julia`.
|
||||
- `cache-artifacts` - Whether to cache the depot's `artifacts` directory. Defaults to `true`.
|
||||
|
||||
35
action.yml
35
action.yml
@@ -9,9 +9,9 @@ branding:
|
||||
inputs:
|
||||
cache-name:
|
||||
description: >-
|
||||
The cache key prefix. Unless disabled the key body automatically includes matrix vars, and the OS.
|
||||
The cache key prefix. The key body automatically includes the OS and, unless disabled, the matrix vars.
|
||||
Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
|
||||
default: julia-cache-${{ github.workflow }}-${{ github.job }}
|
||||
default: julia-cache;workflow=${{ github.workflow }};job=${{ github.job }}
|
||||
include-matrix:
|
||||
description: Whether to include the matrix values when constructing the cache key.
|
||||
default: 'true'
|
||||
@@ -51,6 +51,14 @@ outputs:
|
||||
runs:
|
||||
using: 'composite'
|
||||
steps:
|
||||
- name: Install jq
|
||||
# Skip installation on GitHub-hosted runners:
|
||||
# https://github.com/orgs/community/discussions/48359#discussioncomment-5323864
|
||||
if: ${{ !startsWith(runner.name, 'GitHub Actions') }}
|
||||
uses: dcarbone/install-jq-action@v2.1.0
|
||||
with:
|
||||
force: false # Skip install when an existing `jq` is present
|
||||
|
||||
- id: paths
|
||||
run: |
|
||||
if [ -n "${{ inputs.depot }}" ]; then
|
||||
@@ -78,18 +86,21 @@ runs:
|
||||
env:
|
||||
PATH_DELIMITER: ${{ runner.OS == 'Windows' && ';' || ':' }}
|
||||
|
||||
# MATRIX_STRING is a join of all matrix variables that helps concurrent runs have a unique cache key.
|
||||
# The underscore at the end of the restore key demarks the end of the restore section. Without this
|
||||
# a runner without a matrix has a restore key that will cause impropper clearing of caches from those
|
||||
# with a matrix.
|
||||
- id: keys
|
||||
- name: Generate Keys
|
||||
id: keys
|
||||
run: |
|
||||
[ "${{ inputs.include-matrix }}" == "true" ] && MATRIX_STRING="${{ join(matrix.*, '-') }}"
|
||||
[ -n "$MATRIX_STRING" ] && MATRIX_STRING="-${MATRIX_STRING}"
|
||||
RESTORE_KEY="${{ inputs.cache-name }}-${{ runner.os }}${MATRIX_STRING}-${{ steps.paths.outputs.depot }}_"
|
||||
echo "restore-key=${RESTORE_KEY}" >> $GITHUB_OUTPUT
|
||||
echo "key=${RESTORE_KEY}${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT
|
||||
# `matrix_key` joins all of matrix keys/values (including nested objects) to ensure that concurrent runs each use a unique cache key.
|
||||
# When `matrix` isn't set for the job then `MATRIX_JSON=null`.
|
||||
if [ "${{ inputs.include-matrix }}" == "true" ] && [ "$MATRIX_JSON" != "null" ]; then
|
||||
matrix_key=$(echo "$MATRIX_JSON" | jq 'paths(type != "object") as $p | ($p | join("-")) + "=" + (getpath($p) | tostring)' | jq -rs 'join(";") | . + ";"')
|
||||
fi
|
||||
restore_key="${{ inputs.cache-name }};os=${{ runner.os }};${matrix_key}"
|
||||
key="${restore_key}run_id=${{ github.run_id }};run_attempt=${{ github.run_attempt }}"
|
||||
echo "restore-key=${restore_key}" >> $GITHUB_OUTPUT
|
||||
echo "key=${key}" >> $GITHUB_OUTPUT
|
||||
shell: bash
|
||||
env:
|
||||
MATRIX_JSON: ${{ toJSON(matrix) }}
|
||||
|
||||
- uses: actions/cache@4d4ae6ae148a43d0fd1eda1800170683e9882738
|
||||
id: cache
|
||||
|
||||
@@ -13,7 +13,7 @@ function handle_caches()
|
||||
for _ in 1:5 # limit to avoid accidental rate limiting
|
||||
hits = split(strip(read(`gh cache list --limit 100 --repo $repo`, String)), keepempty=false)
|
||||
search_again = length(hits) == 100
|
||||
filter!(contains(restore_key), hits)
|
||||
filter!(startswith(restore_key), hits)
|
||||
isempty(hits) && break
|
||||
# We can delete everything that matches the restore key because the new cache is saved later.
|
||||
for c in hits
|
||||
|
||||
Reference in New Issue
Block a user