Expand paths which use tilde (#141)

* update any cached registries

* add comment

* handle really old julia versions

* Update action.yml

Co-authored-by: Curtis Vogt <curtis.vogt@gmail.com>

---------

Co-authored-by: Curtis Vogt <curtis.vogt@gmail.com>

.github/workflows/CI.yml

@@ -59,7 +59,7 @@ jobs:
     env:
       JULIA_DEPOT_PATH: /tmp/julia-depot
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - name: Set cache-name
       id: cache-name
       shell: bash
@@ -117,7 +117,7 @@ jobs:
     env:
       JULIA_DEPOT_PATH: /tmp/julia-depot
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.version }}
@@ -161,7 +161,7 @@ jobs:
     outputs:
       cache-name: ${{ steps.cache-name.outputs.cache-name }}
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - name: Set cache-name
       id: cache-name
       run: |
@@ -193,7 +193,7 @@ jobs:
     needs: test-save-nomatrix
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - name: Restore cache
       id: cache
       uses: ./
@@ -233,7 +233,7 @@ jobs:
     outputs:
       cache-name: ${{ steps.cache-name.outputs.cache-name }}
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - name: Set cache-name
       id: cache-name
       run: |
@@ -259,7 +259,7 @@ jobs:
     needs: test-save-cloned-registry
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
     - name: Add General registry clone
       shell: julia --color=yes {0}
       run: |

README.md

@@ -19,29 +19,20 @@ permissions:
 jobs:
   test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        version:
-          - '1.6'
-          - '1'
     steps:
-      - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v2
-      - uses: julia-actions/cache@v2
-        with:
-          cache-name: 'test-${{ matrix.os }}'
-      - uses: julia-actions/julia-buildpkg@v1
-      - uses: julia-actions/julia-runtest@v1
+    - uses: actions/checkout@v4
+    - uses: julia-actions/setup-julia@v2
+    - uses: julia-actions/cache@v2
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-runtest@v1
 ```
 
 By default all depot directories called out below are cached.
 
 ### Requirements
 
-This action uses [`jq`](https://github.com/jqlang/jq) to parse JSON.
-`jq` is installed by default in GitHub-hosted runners.
-[`dcarbone/install-jq-action`](https://github.com/dcarbone/install-jq-action) is used to check that `jq` is available and install it if not.
-**Note:** installing `jq` with `dcarbone/install-jq-action` requires that curl is available; this may not be the case in custom containers.
+- `jq`: This action uses [`jq`](https://github.com/jqlang/jq) to parse JSON. For GitHub-hosted runners, `jq` is installed by default. On self-hosted runners and custom containers, if `jq` is not already available, this action will automatically use [`dcarbone/install-jq-action`](https://github.com/dcarbone/install-jq-action) to install `jq` (Note: `dcarbone/install-jq-action` requires that `curl` is installed; this may not always be the case in custom containers and self-hosted runners).
+- `bash`: This action requires `bash`. For GitHub-hosted runners `bash` is installed by default. Self-hosted runners will need to ensure that `bash` is installed and available on the `PATH`.
 
 ### Optional Inputs
 
@@ -83,8 +74,8 @@ The cache key that the cache will be saved as is based on:
 - The run attempt number
 
 > [!NOTE]
-> If in your workflow if you do not use a matrix for concurrency you should make `cache-name` such that it is unique for
-> concurrent jobs, otherwise caching may not be effective.
+> If there is job concurrency that is not fully defined by a matrix you should ensure that `cache-name` is 
+> unique for each concurrent job, otherwise caching may not be effective.
 
 ### Cache Retention
 

action.yml

@@ -66,6 +66,7 @@ runs:
         else
             depot="~/.julia"
         fi
+        depot="${depot/#\~/$HOME}"  # Expand tilde which cannot be used in BASH checks (i.e. `[ -d "~/.julia" ]` fails)
         echo "depot=$depot" | tee -a "$GITHUB_OUTPUT"
 
         cache_paths=()
@@ -75,7 +76,7 @@ runs:
         [ "${{ inputs.cache-packages }}" = "true" ] && cache_paths+=("$packages_path")
         registries_path="${depot}/registries"
         if [ "${{ inputs.cache-registries }}" = "true" ]; then
-            if [ ! -d "${registries_path/#\~/$HOME}" ]; then
+            if [ ! -d "${registries_path}" ]; then
                 cache_paths+=("$registries_path")
             else
                 echo "::warning::Julia depot registries already exist. Skipping restoring of cached registries to avoid potential merge conflicts when updating. Please ensure that \`julia-actions/cache\` precedes any workflow steps which add registries."
@@ -132,12 +133,43 @@ runs:
         du -shc ${{ steps.paths.outputs.depot }}/* || true
       shell: bash
 
-    # github and actions/cache doesn't provide a way to update a cache at a given key, so we delete any
-    # that match the restore key just before saving the new cache
+    # issue https://github.com/julia-actions/cache/issues/110
+    # Pkg may not run `Registry.update()` if a manifest exists, which may exist because of a
+    # `Pkg.dev` call or because one is added to the repo. So be safe and update cached registries here.
+    # Older (~v1.0) versions of julia that don't have `Pkg.Registry.update()` seem to always update registries in
+    # Pkg operations. So this is only necessary for newer julia versions.
+    - name: Update any cached registries
+      if: ${{ inputs.cache-registries == 'true' }}
+      run: |
+        if [ -d "${{ steps.paths.outputs.depot }}/registries" ] && [ -n "$(ls -A "${{ steps.paths.outputs.depot }}/registries")" ]; then
+          echo "Registries directory exists and is non-empty. Updating any registries"
+          julia -e "import Pkg; isdefined(Pkg, :Registry) && Pkg.Registry.update();"
+        else
+          echo "Registries directory does not exist or is empty. Skipping registry update"
+        fi
+      shell: bash
+
+    # GitHub actions cache entries are immutable and cannot be updated. In order to have both the Julia
+    # depot cache be up-to-date and avoid storing redundant cache entries we'll manually cleanup old
+    # cache entries before the new cache is saved. However, we need to be careful with our manual
+    # cleanup as otherwise we can cause cache misses for jobs which would have normally had a cache hit.
+    # Some scenarios to keep in mind include:
+    #
+    # - Job failures result in the post-action for `actions/cache` being skipped. If we delete all cache
+    #   entries for the branch we may have no cache entry available for the next run.
+    # - We should avoid deleting old cache entries for the default branch since these entries serve as
+    #   the fallback if no earlier cache entry exists on a branch. We can rely on GitHub's default cache
+    #   eviction policy here which will remove the oldest cache entry first.
+    #
+    # References:
+    # - https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+    # - https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#usage-limits-and-eviction-policy
 
     # Not windows
     - uses: pyTooling/Actions/with-post-step@e9d0dc3dba9fda45f195946858708f60c0240caf # v1.0.5
-      if: ${{ inputs.delete-old-caches != 'false' && runner.OS != 'Windows' }}
+      if: ${{ inputs.delete-old-caches != 'false' &&
+              github.ref != format('refs/heads/{0}', github.event.repository.default_branch) &&
+              runner.OS != 'Windows' }}
       with:
         # seems like there has to be a `main` step in this action. Could list caches for info if we wanted
         # main:  julia ${{ github.action_path }}/handle_caches.jl "${{ github.repository }}" "list"
@@ -148,7 +180,9 @@ runs:
 
     # Windows (because this action uses command prompt on windows)
     - uses: pyTooling/Actions/with-post-step@e9d0dc3dba9fda45f195946858708f60c0240caf # v1.0.5
-      if: ${{ inputs.delete-old-caches != 'false' && runner.OS == 'Windows' }}
+      if: ${{ inputs.delete-old-caches != 'false' &&
+              github.ref != format('refs/heads/{0}', github.event.repository.default_branch) &&
+              runner.OS == 'Windows' }}
       with:
         main: echo ""
         post: cd %GITHUB_ACTION_PATH% && julia handle_caches.jl rm "${{ github.repository }}" "${{ steps.keys.outputs.restore-key }}" "${{ github.ref }}" "${{ inputs.delete-old-caches != 'required' }}"

devdocs/making_a_new_release.md

@@ -0,0 +1,56 @@
+# Making a new release
+
+In this guide, as an example, `v2.2.0` refers to the version number of the new release that you want to make.
+
+## Part 1: Use the Git CLI to create and push the Git tags
+
+Step 1: Create a new lightweight tag of the form `vMAJOR.MINOR.PATCH`.
+
+```bash
+git clone git@github.com:julia-actions/cache.git
+cd cache
+git fetch --all --tags
+
+git checkout main
+
+git --no-pager log -1
+# Take note of the commit hash here.
+
+# Now, create a new lightweight tag of the form `vMAJOR.MINOR.PATCH`.
+#
+# Replace `commit_hash` with the commit hash that you obtained from the
+# `git log -1` step.
+#
+# Replace `v2.2.0` with the actual version number that you want to use.
+git tag v2.2.0 commit_hash
+```
+
+Step 2: Once you've created the new release, you need to update the `v2` tag to point to the new release. For example, suppose that the previous release was `v2.1.0`, and suppose that you just created the new release `v2.2.0`. You need to update the `v2` tag so that it points to `v2.2.0`. Here are the commands:
+
+```bash
+# Create/update the new v2 tag locally, where the new v2 tag will point to the
+# release that you created in the previous step.
+#
+# Make sure to change `v2.2.0` to the actual value for the tag that you just
+# created in the previous step.
+#
+# The `-f` flag forcibly overwrites the old
+# `v2` tag (if it exists).
+git tag -f v2 v2.2.0
+```
+
+Step 3: Now you need to push the tags:
+
+```bash
+# Regular-push the new `v2.2.0` tag:
+git push origin tag v2.2.0
+
+# Force-push the new v2 tag:
+git push origin tag v2 --force
+```
+
+## Part 2: Create the GitHub Release
+
+Go to the [Releases](https://github.com/julia-actions/cache/releases) section of this repo and create a new release (using the GitHub web interface).
+
+For the "choose a tag" drop-down field, select the `v2.2.0` tag that you created and pushed in Part 1 of this guide.

handle_caches.jl

@@ -10,37 +10,48 @@ function handle_caches()
         repo, restore_key, ref = ARGS[2:4]
         allow_failure = ARGS[5] == "true"
 
-        endpoint = "/repos/$repo/actions/caches"
         page = 1
         per_page = 100
-        escaped_restore_key = replace(restore_key, "\"" => "\\\"")
-        query = ".actions_caches[] | select(.key | startswith(\"$escaped_restore_key\")) | .id"
-
-        deletions = String[]
-        failures = String[]
+        skipped = String[]
+        deleted = String[]
+        failed = String[]
         while 1 <= page <= 5 # limit to avoid accidental rate limiting
-            cmd = `gh api -X GET $endpoint -F ref=$ref -F per_page=$per_page -F page=$page --jq $query`
+            # https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#list-github-actions-caches-for-a-repository
+            # Note: The `key` field matches on the full key or a prefix.
+            cmd = ```
+                gh api -X GET /repos/$repo/actions/caches
+                    --field per_page=$per_page
+                    --field page=$page
+                    --field ref=$ref
+                    --field key=$restore_key
+                    --field sort=last_accessed_at
+                    --field direction=desc
+                    --jq '.actions_caches[].id'
+                ```
             ids = split(read(cmd, String); keepempty=false)
-            page = length(ids) == per_page ? page + 1 : -1
 
-            # We can delete all cache entries on this branch that matches the restore key
-            # because the new cache is saved later.
+            # Avoid deleting the latest used cache entry. This is particularly important for
+            # job failures where a new cache entry will not be saved after this.
+            page == 1 && !isempty(ids) && push!(skipped, popfirst!(ids))
+
             for id in ids
                 try
                     run(`gh cache delete $id --repo $repo`)
-                    push!(deletions, id)
+                    push!(deleted, id)
                 catch e
                     @error e
-                    push!(failures, id)
+                    push!(failed, id)
                 end
             end
+
+            page = length(ids) == per_page ? page + 1 : -1
         end
-        if isempty(failures) && isempty(deletions)
+        if isempty(skipped) && isempty(deleted) && isempty(failed)
             println("No existing caches found on ref `$ref` matching restore key `$restore_key`")
         else
-            if !isempty(failures)
-                println("Failed to delete $(length(failures)) existing caches on ref `$ref` matching restore key `$restore_key`")
-                println.(failures)
+            if !isempty(failed)
+                println("Failed to delete $(length(failed)) existing caches on ref `$ref` matching restore key `$restore_key`")
+                println.(failed)
                 @info """
                     To delete caches you need to grant the following to the default `GITHUB_TOKEN` by adding
                     this to your workflow:
@@ -55,9 +66,9 @@ function handle_caches()
                     """
                 allow_failure || exit(1)
             end
-            if !isempty(deletions)
-                println("Deleted $(length(deletions)) caches on ref `$ref` matching restore key `$restore_key`")
-                println.(deletions)
+            if !isempty(deleted)
+                println("Deleted $(length(deleted)) caches on ref `$ref` matching restore key `$restore_key`")
+                println.(deleted)
             end
         end
     else