Unique cache-key for job matrix objects (#88)

* Unique cache-key for job matrix objects

* Update workflow to use object in job matrix

* Restore key should always match startswith

* fixup! Unique cache-key for job matrix objects

* Debug no-matrix

* Tests require overriding workflow/job

* Skip generating matrix_key when no matrix is used

* Install jq for self-hosted runners

* fixup! Install jq for self-hosted runners

* Skip install when not needed

* fixup! Skip install when not needed

* fixup! Skip install when not needed

* Improve `cache-name` description

Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>

* Update  description in README

* Use actions/checkout@v4 in example

* add missing period

---------

Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>

.github/workflows/CI.yml

@@ -36,7 +36,13 @@ jobs:
     needs: generate-key
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        dep:
+          - name: pandoc_jll
+            version: "3"
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macOS-latest
       fail-fast: false
     runs-on: ${{ matrix.os }}
     env:
@@ -55,7 +61,7 @@ jobs:
         @assert !isdir(dir)
     - name: Install a small binary
       shell: 'julia --color=yes {0}'
-      run: 'using Pkg; Pkg.add("pandoc_jll")'
+      run: 'using Pkg; Pkg.add(PackageSpec(name="${{ matrix.dep.name }}", version="${{ matrix.dep.version }}"))'
 
   # Do tests with no matrix also given the matrix is auto-included in cache key
   test-save-nomatrix:
@@ -81,7 +87,13 @@ jobs:
     needs: [generate-key, test-save]
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        dep:
+          - name: pandoc_jll
+            version: "3"
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macOS-latest
       fail-fast: false
     runs-on: ${{ matrix.os }}
     env:

README.md

@@ -20,7 +20,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: julia-actions/setup-julia@v1
     - uses: julia-actions/cache@v1
     - uses: julia-actions/julia-buildpkg@v1
@@ -31,7 +31,7 @@ By default all depot directories called out below are cached.
 
 ### Optional Inputs
 
-- `cache-name` - The cache key prefix. Defaults to `julia-cache-${{ github.workflow }}-${{ github.job }}`. The key body automatically includes matrix vars and the OS. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
+- `cache-name` - The cache key prefix. Defaults to `julia-cache;workflow=${{ github.workflow }};job=${{ github.job }}`. The key body automatically includes the OS and, unless disabled with `include-matrix`, the matrix vars. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
 - `include-matrix` - Whether to include the matrix values when constructing the cache key. Defaults to `true`.
 - `depot` - Path to a Julia [depot](https://pkgdocs.julialang.org/v1/glossary/) directory where cached data will be saved to and restored from. Defaults to the first depot in [`JULIA_DEPOT_PATH`](https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH) if specified. Otherwise, defaults to `~/.julia`.
 - `cache-artifacts` - Whether to cache the depot's `artifacts` directory. Defaults to `true`.

action.yml

@@ -9,9 +9,9 @@ branding:
 inputs:
   cache-name:
     description: >-
-      The cache key prefix. Unless disabled the key body automatically includes matrix vars, and the OS.
+      The cache key prefix. The key body automatically includes the OS and, unless disabled, the matrix vars.
       Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
-    default: julia-cache-${{ github.workflow }}-${{ github.job }}
+    default: julia-cache;workflow=${{ github.workflow }};job=${{ github.job }}
   include-matrix:
     description: Whether to include the matrix values when constructing the cache key.
     default: 'true'
@@ -51,6 +51,14 @@ outputs:
 runs:
   using: 'composite'
   steps:
+    - name: Install jq
+      # Skip installation on GitHub-hosted runners:
+      # https://github.com/orgs/community/discussions/48359#discussioncomment-5323864
+      if: ${{ !startsWith(runner.name, 'GitHub Actions') }}
+      uses: dcarbone/install-jq-action@v2.1.0
+      with:
+        force: false  # Skip install when an existing `jq` is present
+
     - id: paths
       run: |
         if [ -n "${{ inputs.depot }}" ]; then
@@ -78,18 +86,21 @@ runs:
       env:
         PATH_DELIMITER: ${{ runner.OS == 'Windows' && ';' || ':' }}
 
-      # MATRIX_STRING is a join of all matrix variables that helps concurrent runs have a unique cache key.
+    - name: Generate Keys
-      # The underscore at the end of the restore key demarks the end of the restore section. Without this
+      id: keys
-      # a runner without a matrix has a restore key that will cause impropper clearing of caches from those
-      # with a matrix.
-    - id: keys
       run: |
-        [ "${{ inputs.include-matrix }}" == "true" ] && MATRIX_STRING="${{ join(matrix.*, '-') }}"
+        # `matrix_key` joins all of matrix keys/values (including nested objects) to ensure that concurrent runs each use a unique cache key.
-        [ -n "$MATRIX_STRING" ] && MATRIX_STRING="-${MATRIX_STRING}"
+        # When `matrix` isn't set for the job then `MATRIX_JSON=null`.
-        RESTORE_KEY="${{ inputs.cache-name }}-${{ runner.os }}${MATRIX_STRING}-${{ steps.paths.outputs.depot }}_"
+        if [ "${{ inputs.include-matrix }}" == "true" ] && [ "$MATRIX_JSON" != "null" ]; then
-        echo "restore-key=${RESTORE_KEY}" >> $GITHUB_OUTPUT
+          matrix_key=$(echo "$MATRIX_JSON" | jq 'paths(type != "object") as $p | ($p | join("-")) + "=" + (getpath($p) | tostring)' | jq -rs 'join(";") | . + ";"')
-        echo "key=${RESTORE_KEY}${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT
+        fi
+        restore_key="${{ inputs.cache-name }};os=${{ runner.os }};${matrix_key}"
+        key="${restore_key}run_id=${{ github.run_id }};run_attempt=${{ github.run_attempt }}"
+        echo "restore-key=${restore_key}" >> $GITHUB_OUTPUT
+        echo "key=${key}" >> $GITHUB_OUTPUT
       shell: bash
+      env:
+        MATRIX_JSON: ${{ toJSON(matrix) }}
 
     - uses: actions/cache@4d4ae6ae148a43d0fd1eda1800170683e9882738
       id: cache

handle_caches.jl

@@ -13,7 +13,7 @@ function handle_caches()
         for _ in 1:5 # limit to avoid accidental rate limiting
             hits = split(strip(read(`gh cache list --limit 100 --repo $repo`, String)), keepempty=false)
             search_again = length(hits) == 100
-            filter!(contains(restore_key), hits)
+            filter!(startswith(restore_key), hits)
             isempty(hits) && break
             # We can delete everything that matches the restore key because the new cache is saved later.
             for c in hits