URL encode any invalid key characters (#93)

* URL encode any invalid key characters

* Test we handle invalid chars

* Job matrices must match

* Empty commit

* Empty commit

.github/workflows/CI.yml

@@ -36,9 +36,18 @@ jobs:
     needs: generate-key
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        dep:
+          - name: pandoc_jll
+            version: "3"
+            invalid-chars: ","  # Use invalid characters in job matrix to ensure we escape them
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macOS-latest
       fail-fast: false
     runs-on: ${{ matrix.os }}
+    env:
+      JULIA_DEPOT_PATH: /tmp/julia-depot
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
     - name: Save cache
@@ -53,7 +62,7 @@ jobs:
         @assert !isdir(dir)
     - name: Install a small binary
       shell: 'julia --color=yes {0}'
-      run: 'using Pkg; Pkg.add("pandoc_jll")'
+      run: 'using Pkg; Pkg.add(PackageSpec(name="${{ matrix.dep.name }}", version="${{ matrix.dep.version }}"))'
 
   # Do tests with no matrix also given the matrix is auto-included in cache key
   test-save-nomatrix:
@@ -79,9 +88,18 @@ jobs:
     needs: [generate-key, test-save]
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
+        dep:
+          - name: pandoc_jll
+            version: "3"
+            invalid-chars: ","  # Use invalid characters in job matrix to ensure we escape them
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macOS-latest
       fail-fast: false
     runs-on: ${{ matrix.os }}
+    env:
+      JULIA_DEPOT_PATH: /tmp/julia-depot
     steps:
     - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
     - name: Restore cache

README.md

@@ -20,35 +20,28 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: julia-actions/setup-julia@v1
     - uses: julia-actions/cache@v1
     - uses: julia-actions/julia-buildpkg@v1
     - uses: julia-actions/julia-runtest@v1
 ```
 
-By default the majority of the depot is cached. To also cache `~/.julia/registries/`, use
-
-```yaml
-    - uses: julia-actions/cache@v1
-      with:
-        cache-registries: "true"
-```
-
-However note that caching the registries may mean that the registry will not be updated each run.
+By default all depot directories called out below are cached.
 
 ### Optional Inputs
 
-- `cache-name` - The cache key prefix. Defaults to `julia-cache-${{ github.workflow }}-${{ github.job }}`. The key body automatically includes matrix vars and the OS. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
+- `cache-name` - The cache key prefix. Defaults to `julia-cache;workflow=${{ github.workflow }};job=${{ github.job }}`. The key body automatically includes the OS and, unless disabled with `include-matrix`, the matrix vars. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
 - `include-matrix` - Whether to include the matrix values when constructing the cache key. Defaults to `true`.
-- `cache-artifacts` - Whether to cache `~/.julia/artifacts/`. Defaults to `true`.
-- `cache-packages` - Whether to cache `~/.julia/packages/`. Defaults to `true`.
-- `cache-registries` - Whether to cache `~/.julia/registries/`. Defaults to `false`. Disabled to ensure CI gets latest versions.
-- `cache-compiled` - Whether to cache `~/.julia/compiled/`. Defaults to `true`.
-- `cache-scratchspaces` - Whether to cache `~/.julia/scratchspaces/`. Defaults to `true`.
-- `cache-logs` - Whether to cache `~/.julia/logs/`. Defaults to `true`. Helps auto-`Pkg.gc()` keep the cache small.
-- `delete-old-caches` - Whether to delete old caches for the given key. Defaults to `true`
-- `token` - A github PAT. Defaults to `github.token`. Requires `repo` scope to enable the deletion of old caches.
+- `depot` - Path to a Julia [depot](https://pkgdocs.julialang.org/v1/glossary/) directory where cached data will be saved to and restored from. Defaults to the first depot in [`JULIA_DEPOT_PATH`](https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH) if specified. Otherwise, defaults to `~/.julia`.
+- `cache-artifacts` - Whether to cache the depot's `artifacts` directory. Defaults to `true`.
+- `cache-packages` - Whether to cache the depot's `packages` directory. Defaults to `true`.
+- `cache-registries` - Whether to cache the depot's `registries` directory. Defaults to `true`.
+- `cache-compiled` - Whether to cache the depot's `compiled` directory. Defaults to `true`.
+- `cache-scratchspaces` - Whether to cache the depot's `scratchspaces` directory. Defaults to `true`.
+- `cache-logs` - Whether to cache the depot's `logs` directory. Defaults to `true`. Helps auto-`Pkg.gc()` keep the cache small.
+- `delete-old-caches` - Whether to delete old caches for the given key. Defaults to `true`.
+- `token` - A [GitHub PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens). Defaults to `github.token`. Requires `repo` scope to enable the deletion of old caches.
 
 ### Outputs
 
@@ -82,16 +75,16 @@ This action automatically deletes old caches that match the first 4 fields of th
 - All variables in the `matrix` (unless disabled via `include-matrix: 'false'`)
 - The `runner.os` (may be in the matrix too, but included for safety)
 
-Which means your caches files will not grow needlessly. Github also deletes cache files after
+Which means your caches files will not grow needlessly. GitHub also deletes cache files after
 [90 days which can be increased in private repos to up to 400 days](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization)
 
 > [!NOTE]
-> To allow deletion of caches you will likely need to grant the following to the default
-> `GITHUB_TOKEN` by adding this to your yml:
-> ```
+> To allow deletion of caches you will likely need to [grant the following permissions](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs)
+> to the `GITHUB_TOKEN` by adding this to your GitHub actions workflow:
+> ```yaml
 > permissions:
->     actions: write
->     contents: read
+>   actions: write
+>   contents: read
 > ```
 > (Note this won't work for fork PRs but should once merged)
 > Or provide a token with `repo` scope via the `token` input option.

action.yml

@@ -8,72 +8,102 @@ branding:
 
 inputs:
   cache-name:
-    description: 'The cache key prefix. Unless disabled the key body automatically includes matrix vars, and the OS. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.'
-    default: 'julia-cache-${{ github.workflow }}-${{ github.job }}'
+    description: >-
+      The cache key prefix. The key body automatically includes the OS and, unless disabled, the matrix vars.
+      Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
+    default: julia-cache;workflow=${{ github.workflow }};job=${{ github.job }}
   include-matrix:
-    description: 'Whether to include the matrix values when constructing the cache key'
+    description: Whether to include the matrix values when constructing the cache key.
     default: 'true'
+  depot:
+    description: Path to a Julia depot directory where cached data will be saved to and restored from.
+    default: ''
   cache-artifacts:
-    description: 'Whether to cache ~/.julia/artifacts/'
+    description: Whether to cache the depot's `artifacts` directory.
     default: 'true'
   cache-packages:
-    description: 'Whether to cache ~/.julia/packages/'
+    description: Whether to cache the depot's `packages` directory.
     default: 'true'
   cache-registries:
-    description: 'Whether to cache ~/.julia/registries/. This is off by default to ensure CI gets latest versions'
-    default: 'false'
+    description: Whether to cache the depot's `registries` directory.
+    default: 'true'
   cache-compiled:
-    description: 'Whether to cache ~/.julia/compiled/'
+    description: Whether to cache the depot's `compiled` directory.
     default: 'true'
   cache-scratchspaces:
-    description: 'Whether to cache ~/.julia/scratchspaces/'
+    description: Whether to cache the depot's `scratchspaces` directory.
     default: 'true'
   cache-logs:
-    description: 'Whether to cache ~/.julia/logs/. This helps automatic Pkg.gc() keep the cache size down'
+    description: Whether to cache the depot's `logs` directory. This helps automatic `Pkg.gc()` keep the cache size down.
     default: 'true'
   delete-old-caches:
-    description: 'Whether to delete old caches for the given key'
+    description: Whether to delete old caches for the given key.
     default: 'true'
   token:
-    description: 'A github PAT. Requires `repo` scope to enable the deletion of old caches'
-    default: '${{ github.token }}'
+    description: A GitHub PAT. Requires `repo` scope to enable the deletion of old caches.
+    default: ${{ github.token }}
 
 outputs:
   cache-hit:
-    description: 'A boolean value to indicate an exact match was found for the primary key. Returns \"\" when the key is new. Forwarded from actions/cache'
+    description: A boolean value to indicate an exact match was found for the primary key. Returns "" when the key is new. Forwarded from actions/cache.
     value: ${{ steps.hit.outputs.cache-hit }}
 
 runs:
   using: 'composite'
   steps:
+    - name: Install jq
+      # Skip installation on GitHub-hosted runners:
+      # https://github.com/orgs/community/discussions/48359#discussioncomment-5323864
+      if: ${{ !startsWith(runner.name, 'GitHub Actions') }}
+      uses: dcarbone/install-jq-action@v2.1.0
+      with:
+        force: false  # Skip install when an existing `jq` is present
+
     - id: paths
       run: |
-        [ "${{ inputs.cache-artifacts }}" = "true" ] && A_PATH="~/.julia/artifacts"
+        if [ -n "${{ inputs.depot }}" ]; then
+            depot="${{ inputs.depot }}"
+        elif [ -n "$JULIA_DEPOT_PATH" ]; then
+            # Use the first depot path
+            depot=$(echo $JULIA_DEPOT_PATH | cut -d$PATH_DELIMITER -f1)
+        else
+            depot="~/.julia"
+        fi
+        echo "depot=$depot" >> $GITHUB_OUTPUT
+        [ "${{ inputs.cache-artifacts }}" = "true" ] && A_PATH="${depot}/artifacts"
         echo "artifacts-path=$A_PATH" >> $GITHUB_OUTPUT
-        [ "${{ inputs.cache-packages }}" = "true" ] && P_PATH="~/.julia/packages"
+        [ "${{ inputs.cache-packages }}" = "true" ] && P_PATH="${depot}/packages"
         echo "packages-path=$P_PATH" >> $GITHUB_OUTPUT
-        [ "${{ inputs.cache-registries }}" = "true" ] && R_PATH="~/.julia/registries"
+        [ "${{ inputs.cache-registries }}" = "true" ] && R_PATH="${depot}/registries"
         echo "registries-path=$R_PATH" >> $GITHUB_OUTPUT
-        [ "${{ inputs.cache-compiled }}" = "true" ] && PCC_PATH="~/.julia/compiled"
+        [ "${{ inputs.cache-compiled }}" = "true" ] && PCC_PATH="${depot}/compiled"
         echo "compiled-path=$PCC_PATH" >> $GITHUB_OUTPUT
-        [ "${{ inputs.cache-scratchspaces }}" = "true" ] && S_PATH="~/.julia/scratchspaces"
+        [ "${{ inputs.cache-scratchspaces }}" = "true" ] && S_PATH="${depot}/scratchspaces"
         echo "scratchspaces-path=$S_PATH" >> $GITHUB_OUTPUT
-        [ "${{ inputs.cache-logs }}" = "true" ] && L_PATH="~/.julia/logs"
+        [ "${{ inputs.cache-logs }}" = "true" ] && L_PATH="${depot}/logs"
         echo "logs-path=$L_PATH" >> $GITHUB_OUTPUT
       shell: bash
+      env:
+        PATH_DELIMITER: ${{ runner.OS == 'Windows' && ';' || ':' }}
 
-      # MATRIX_STRING is a join of all matrix variables that helps concurrent runs have a unique cache key.
-      # The underscore at the end of the restore key demarks the end of the restore section. Without this
-      # a runner without a matrix has a restore key that will cause impropper clearing of caches from those
-      # with a matrix.
-    - id: keys
+    - name: Generate Keys
+      id: keys
       run: |
-        [ "${{ inputs.include-matrix }}" == "true" ] && MATRIX_STRING="${{ join(matrix.*, '-') }}"
-        [ -n "$MATRIX_STRING" ] && MATRIX_STRING="-${MATRIX_STRING}"
-        RESTORE_KEY="${{ inputs.cache-name }}-${{ runner.os }}${MATRIX_STRING}_"
-        echo "restore-key=${RESTORE_KEY}" >> $GITHUB_OUTPUT
-        echo "key=${RESTORE_KEY}${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT
+        # `matrix_key` joins all of matrix keys/values (including nested objects) to ensure that concurrent runs each use a unique cache key.
+        # When `matrix` isn't set for the job then `MATRIX_JSON=null`.
+        if [ "${{ inputs.include-matrix }}" == "true" ] && [ "$MATRIX_JSON" != "null" ]; then
+          matrix_key=$(echo "$MATRIX_JSON" | jq 'paths(type != "object") as $p | ($p | join("-")) + "=" + (getpath($p) | tostring)' | jq -rs 'join(";") | . + ";"')
+        fi
+        restore_key="${{ inputs.cache-name }};os=${{ runner.os }};${matrix_key}"
+        # URL encode any restricted characters:
+        # https://github.com/actions/toolkit/blob/5430c5d84832076372990c7c27f900878ff66dc9/packages/cache/src/cache.ts#L38-L43
+        restore_key=$(sed 's/,/%2C/g' <<<"${restore_key}")
+        key="${restore_key}run_id=${{ github.run_id }};run_attempt=${{ github.run_attempt }}"
+        echo "restore-key=${restore_key}" >> $GITHUB_OUTPUT
+        echo "key=${key}" >> $GITHUB_OUTPUT
       shell: bash
+      env:
+        MATRIX_JSON: ${{ toJSON(matrix) }}
 
     - uses: actions/cache@4d4ae6ae148a43d0fd1eda1800170683e9882738
       id: cache
@@ -91,7 +121,8 @@ runs:
         enableCrossOsArchive: false
 
     - name: list restored depot directory sizes
-      run: du -shc ~/.julia/* || true
+      if: ${{ steps.cache.outputs.cache-hit == 'true' }}
+      run: du -shc ${{ steps.paths.outputs.depot }}/* || true
       shell: bash
 
     # github and actions/cache doesn't provide a way to update a cache at a given key, so we delete any

handle_caches.jl

@@ -13,7 +13,7 @@ function handle_caches()
         for _ in 1:5 # limit to avoid accidental rate limiting
             hits = split(strip(read(`gh cache list --limit 100 --repo $repo`, String)), keepempty=false)
             search_again = length(hits) == 100
-            filter!(contains(restore_key), hits)
+            filter!(startswith(restore_key), hits)
             isempty(hits) && break
             # We can delete everything that matches the restore key because the new cache is saved later.
             for c in hits