Fix incorrect GitHub actions path on containers (#78)

Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+
+updates:
+
+  # Keep dependencies for GitHub Actions up-to-date
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'daily'

.github/workflows/CI.yml

@@ -0,0 +1,145 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'action.yml'
+      - 'handle_caches.jl'
+      - '.github/**'
+  pull_request:
+    paths:
+      - 'action.yml'
+      - 'handle_caches.jl'
+      - '.github/**'
+
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
+
+jobs:
+  generate-key:
+    runs-on: ubuntu-latest
+    outputs:
+      cache-name: ${{ steps.name.outputs.cache-name }}
+    steps:
+    - name: Generate random file
+      shell: 'julia --color=yes {0}'
+      run: 'write("random.txt", string(rand(10)))'
+    - name: Set cache-name as output
+      id: name
+      run: echo "cache-name=${{ hashFiles('random.txt') }}" >> $GITHUB_OUTPUT
+
+  test-save:
+    needs: generate-key
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - name: Save cache
+      id: cache
+      uses: ./
+      with:
+        cache-name: ${{ needs.generate-key.outputs.cache-name }}
+    - name: Check no artifacts dir
+      shell: 'julia --color=yes {0}'
+      run: |
+        dir = joinpath(first(DEPOT_PATH), "artifacts")
+        @assert !isdir(dir)
+    - name: Install a small binary
+      shell: 'julia --color=yes {0}'
+      run: 'using Pkg; Pkg.add("pandoc_jll")'
+
+  # Do tests with no matrix also given the matrix is auto-included in cache key
+  test-save-nomatrix:
+    needs: generate-key
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - name: Save cache
+      id: cache
+      uses: ./
+      with:
+        cache-name: ${{ needs.generate-key.outputs.cache-name }}
+    - name: Check no artifacts dir
+      shell: 'julia --color=yes {0}'
+      run: |
+        dir = joinpath(first(DEPOT_PATH), "artifacts")
+        @assert !isdir(dir)
+    - name: Install a small binary
+      shell: 'julia --color=yes {0}'
+      run: 'using Pkg; Pkg.add("pandoc_jll")'
+
+  test-restore:
+    needs: [generate-key, test-save]
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - name: Restore cache
+      id: cache
+      uses: ./
+      with:
+        cache-name: ${{ needs.generate-key.outputs.cache-name }}
+    - name: Test cache-hit output
+      shell: 'julia --color=yes {0}'
+      run: |
+        @show ENV["cache-hit"]
+        @assert ENV["cache-hit"] == "true"
+      env:
+        cache-hit: ${{ steps.cache.outputs.cache-hit }}
+    - name: Check existance or emptiness of affected dirs
+      shell: 'julia --color=yes {0}'
+      run: |
+        # These dirs should exist as they've been cached
+        artifacts_dir = joinpath(first(DEPOT_PATH), "artifacts")
+        @assert !isempty(readdir(artifacts_dir))
+        packages_dir = joinpath(first(DEPOT_PATH), "packages")
+        @assert !isempty(readdir(packages_dir))
+        compiled_dir = joinpath(first(DEPOT_PATH), "compiled")
+        @assert !isempty(readdir(compiled_dir))
+        scratchspaces_dir = joinpath(first(DEPOT_PATH), "scratchspaces")
+        @assert !isempty(readdir(scratchspaces_dir))
+        logs_dir = joinpath(first(DEPOT_PATH), "logs")
+        @assert !isempty(readdir(logs_dir))
+
+  test-restore-nomatrix:
+    needs: [generate-key, test-save-nomatrix]
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+    - name: Restore cache
+      id: cache
+      uses: ./
+      with:
+        cache-name: ${{ needs.generate-key.outputs.cache-name }}
+    - name: Test cache-hit output
+      shell: 'julia --color=yes {0}'
+      run: |
+        @show ENV["cache-hit"]
+        @assert ENV["cache-hit"] == "true"
+      env:
+        cache-hit: ${{ steps.cache.outputs.cache-hit }}
+    - name: Check existance or emptiness of affected dirs
+      shell: 'julia --color=yes {0}'
+      run: |
+        # These dirs should exist as they've been cached
+        artifacts_dir = joinpath(first(DEPOT_PATH), "artifacts")
+        @assert !isempty(readdir(artifacts_dir))
+        packages_dir = joinpath(first(DEPOT_PATH), "packages")
+        @assert !isempty(readdir(packages_dir))
+        compiled_dir = joinpath(first(DEPOT_PATH), "compiled")
+        @assert !isempty(readdir(compiled_dir))
+        scratchspaces_dir = joinpath(first(DEPOT_PATH), "scratchspaces")
+        @assert !isempty(readdir(scratchspaces_dir))
+        logs_dir = joinpath(first(DEPOT_PATH), "logs")
+        @assert !isempty(readdir(logs_dir))
+

README.md

@@ -1,63 +1,128 @@
-# cache-artifacts
+# julia-actions/cache Action
-A shortcut action to cache Julia artifacts.
 
-Using this action is equivalent to including the following step in your workflows:
+A shortcut action to cache Julia depot contents to reduce GitHub Actions running time.
-
-```yaml
-- uses: actions/cache@v1
-  env:
-    cache-name: cache-artifacts
-  with:
-    path: ~/.julia/artifacts
-    key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-    restore-keys: |
-      ${{ runner.os }}-test-${{ env.cache-name }}-
-      ${{ runner.os }}-test-
-      ${{ runner.os }}-
-```
 
 ## Usage
 
-### Inputs
+An example workflow that uses this action might look like this:
 
 ```yaml
-- uses: julia-actions/cache-artifacts@v1
+name: CI
-  with:
+
-    # The cache name is used as part of the cache key.
+on: [push, pull_request]
-    # It is equivalent to the cache-name environment variable in the snippet above.
+
-    #
+# needed to allow julia-actions/cache to delete old caches that it has created
-    # Default: cache-artifacts
+permissions:
-    cache-name: ''
+  actions: write
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/cache@v1
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-runtest@v1
 ```
 
+By default the majority of the depot is cached. To also cache `~/.julia/registries/`, use
+
+```yaml
+    - uses: julia-actions/cache@v1
+      with:
+        cache-registries: "true"
+```
+
+However note that caching the registries may mean that the registry will not be updated each run.
+
+### Optional Inputs
+
+- `cache-name` - The cache key prefix. Defaults to `julia-cache-${{ github.workflow }}-${{ github.job }}`. The key body automatically includes matrix vars and the OS. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.
+- `include-matrix` - Whether to include the matrix values when constructing the cache key. Defaults to `true`.
+- `cache-artifacts` - Whether to cache `~/.julia/artifacts/`. Defaults to `true`.
+- `cache-packages` - Whether to cache `~/.julia/packages/`. Defaults to `true`.
+- `cache-registries` - Whether to cache `~/.julia/registries/`. Defaults to `false`. Disabled to ensure CI gets latest versions.
+- `cache-compiled` - Whether to cache `~/.julia/compiled/`. Defaults to `true`.
+- `cache-scratchspaces` - Whether to cache `~/.julia/scratchspaces/`. Defaults to `true`.
+- `cache-logs` - Whether to cache `~/.julia/logs/`. Defaults to `true`. Helps auto-`Pkg.gc()` keep the cache small.
+- `delete-old-caches` - Whether to delete old caches for the given key. Defaults to `true`
+- `token` - A github PAT. Defaults to `github.token`. Requires `repo` scope to enable the deletion of old caches.
+
 ### Outputs
 
-```yaml
+- `cache-hit` - A boolean value to indicate an exact match was found for the primary key. Returns \"\" when the key is new. Forwarded from actions/cache.
-outputs:
+
-  # A boolean value to indicate an exact match was found for the primary key.
+## How It Works
-  # Forwarded from actions/cache, check its documentation for more info.
+
-  cache-hit: ''
+This action is a wrapper around <https://github.com/actions/cache>.
-```
+In summary, this action stores the files in the aforementioned paths in one compressed file when running for the first time.
+This cached file is then restored upon the second run, and afterwards resaved under a new key, and the previous cache deleted.
+The benefit of caching is that downloading one big file is quicker than downloading many different files from many different locations
+and precompiling them.
+
+### Cache keys
+
+The cache key that the cache will be saved as is based on:
+- The `cache-name` input
+- All variables in the `matrix` (unless disabled via `include-matrix: 'false'`)
+- The `runner.os` (may be in the matrix too, but included for safety)
+- The run id
+- The run attempt number
+
+> [!NOTE]
+> If in your workflow if you do not use a matrix for concurrency you should make `cache-name` such that it is unique for
+> concurrent jobs, otherwise caching may not be effective.
+
+### Cache Retention
+
+This action automatically deletes old caches that match the first 4 fields of the above key:
+- The `cache-name` input
+- All variables in the `matrix` (unless disabled via `include-matrix: 'false'`)
+- The `runner.os` (may be in the matrix too, but included for safety)
+
+Which means your caches files will not grow needlessly. Github also deletes cache files after
+[90 days which can be increased in private repos to up to 400 days](https://docs.github.com/en/organizations/managing-organization-settings/configuring-the-retention-period-for-github-actions-artifacts-and-logs-in-your-organization)
+
+> [!NOTE]
+> To allow deletion of caches you will likely need to grant the following to the default
+> `GITHUB_TOKEN` by adding this to your yml:
+> ```
+> permissions:
+>     actions: write
+>     contents: read
+> ```
+> (Note this won't work for fork PRs but should once merged)
+> Or provide a token with `repo` scope via the `token` input option.
+> See https://cli.github.com/manual/gh_cache_delete
+
+To disable deletion set input `delete-old-caches: 'false'`.
+
+### Cache Garbage Collection
+
+Caches are restored and re-saved after every run, retaining the state of the depot throughout runs.
+Their size will be regulated like a local depot automatically by the automatic `Pkg.gc()` functionality that
+clears out old content, which is made possible because the `/log` contents are cached.
 
 ## Third Party Notice
 
 This action is built around [`actions/cache`](https://github.com/actions/cache/) and includes parts of that action. `actions/cache` has been released under the following licence:
 
-
 > The MIT License (MIT)
-> 
+>
 > Copyright (c) 2018 GitHub, Inc. and contributors
-> 
+>
 > Permission is hereby granted, free of charge, to any person obtaining a copy
 > of this software and associated documentation files (the "Software"), to deal
 > in the Software without restriction, including without limitation the rights
 > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 > copies of the Software, and to permit persons to whom the Software is
 > furnished to do so, subject to the following conditions:
-> 
+>
 > The above copyright notice and this permission notice shall be included in
 > all copies or substantial portions of the Software.
-> 
+>
 > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE

action.yml

@@ -1,6 +1,6 @@
-name: 'Cache Julia Artifacts'
+name: 'Cache Julia artifacts, packages and registry'
-description: 'Cache Julia artifacts using actions/cache'
+description: 'Cache Julia using actions/cache'
-author: 'Sascha Mann'
+author: 'Sascha Mann, Rik Huijzer, and contributors'
 
 branding:
   icon: 'archive'
@@ -8,27 +8,117 @@ branding:
 
 inputs:
   cache-name:
-    description: 'Name used as part of the cache keys'
+    description: 'The cache key prefix. Unless disabled the key body automatically includes matrix vars, and the OS. Include any other parameters/details in this prefix to ensure one unique cache key per concurrent job type.'
-    default: 'cache-artifacts'
+    default: 'julia-cache-${{ github.workflow }}-${{ github.job }}'
+  include-matrix:
+    description: 'Whether to include the matrix values when constructing the cache key'
+    default: 'true'
+  cache-artifacts:
+    description: 'Whether to cache ~/.julia/artifacts/'
+    default: 'true'
+  cache-packages:
+    description: 'Whether to cache ~/.julia/packages/'
+    default: 'true'
+  cache-registries:
+    description: 'Whether to cache ~/.julia/registries/. This is off by default to ensure CI gets latest versions'
+    default: 'false'
+  cache-compiled:
+    description: 'Whether to cache ~/.julia/compiled/'
+    default: 'true'
+  cache-scratchspaces:
+    description: 'Whether to cache ~/.julia/scratchspaces/'
+    default: 'true'
+  cache-logs:
+    description: 'Whether to cache ~/.julia/logs/. This helps automatic Pkg.gc() keep the cache size down'
+    default: 'true'
+  delete-old-caches:
+    description: 'Whether to delete old caches for the given key'
+    default: 'true'
+  token:
+    description: 'A github PAT. Requires `repo` scope to enable the deletion of old caches'
+    default: '${{ github.token }}'
 
 outputs:
   cache-hit:
-    description: 'A boolean value to indicate an exact match was found for the primary key. Forwarded from actions/cache'
+    description: 'A boolean value to indicate an exact match was found for the primary key. Returns \"\" when the key is new. Forwarded from actions/cache'
+    value: ${{ steps.hit.outputs.cache-hit }}
 
 runs:
   using: 'composite'
   steps:
-    - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed
+    - id: paths
+      run: |
+        [ "${{ inputs.cache-artifacts }}" = "true" ] && A_PATH="~/.julia/artifacts"
+        echo "artifacts-path=$A_PATH" >> $GITHUB_OUTPUT
+        [ "${{ inputs.cache-packages }}" = "true" ] && P_PATH="~/.julia/packages"
+        echo "packages-path=$P_PATH" >> $GITHUB_OUTPUT
+        [ "${{ inputs.cache-registries }}" = "true" ] && R_PATH="~/.julia/registries"
+        echo "registries-path=$R_PATH" >> $GITHUB_OUTPUT
+        [ "${{ inputs.cache-compiled }}" = "true" ] && PCC_PATH="~/.julia/compiled"
+        echo "compiled-path=$PCC_PATH" >> $GITHUB_OUTPUT
+        [ "${{ inputs.cache-scratchspaces }}" = "true" ] && S_PATH="~/.julia/scratchspaces"
+        echo "scratchspaces-path=$S_PATH" >> $GITHUB_OUTPUT
+        [ "${{ inputs.cache-logs }}" = "true" ] && L_PATH="~/.julia/logs"
+        echo "logs-path=$L_PATH" >> $GITHUB_OUTPUT
+      shell: bash
+
+      # MATRIX_STRING is a join of all matrix variables that helps concurrent runs have a unique cache key.
+      # The underscore at the end of the restore key demarks the end of the restore section. Without this
+      # a runner without a matrix has a restore key that will cause impropper clearing of caches from those
+      # with a matrix.
+    - id: keys
+      run: |
+        [ "${{ inputs.include-matrix }}" == "true" ] && MATRIX_STRING="${{ join(matrix.*, '-') }}"
+        [ -n "$MATRIX_STRING" ] && MATRIX_STRING="-${MATRIX_STRING}"
+        RESTORE_KEY="${{ inputs.cache-name }}-${{ runner.os }}${MATRIX_STRING}_"
+        echo "restore-key=${RESTORE_KEY}" >> $GITHUB_OUTPUT
+        echo "key=${RESTORE_KEY}${{ github.run_id }}-${{ github.run_attempt }}" >> $GITHUB_OUTPUT
+      shell: bash
+
+    - uses: actions/cache@4d4ae6ae148a43d0fd1eda1800170683e9882738
       id: cache
       with:
-        path: ~/.julia/artifacts
+        path: |
-        key: ${{ runner.os }}-test-${{ inputs.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          ${{ steps.paths.outputs.artifacts-path }}
-        restore-keys: |
+          ${{ steps.paths.outputs.packages-path }}
-          ${{ runner.os }}-test-${{ inputs.cache-name }}-
+          ${{ steps.paths.outputs.registries-path }}
-          ${{ runner.os }}-test-
+          ${{ steps.paths.outputs.scratchspaces-path }}
-          ${{ runner.os }}-
+          ${{ steps.paths.outputs.compiled-path }}
+          ${{ steps.paths.outputs.logs-path }}
 
-    - run: echo "::set-output name=cache-hit::$CACHE_HIT"
+        key: ${{ steps.keys.outputs.key }}
+        restore-keys: ${{ steps.keys.outputs.restore-key }}
+        enableCrossOsArchive: false
+
+    - name: list restored depot directory sizes
+      run: du -shc ~/.julia/* || true
+      shell: bash
+
+    # github and actions/cache doesn't provide a way to update a cache at a given key, so we delete any
+    # that match the restore key just before saving the new cache
+
+    # Not windows
+    - uses: pyTooling/Actions/with-post-step@adef08d3bdef092282614f3b683897cefae82ee3
+      if: ${{ inputs.delete-old-caches == 'true' && runner.OS != 'Windows' }}
+      with:
+        # seems like there has to be a `main` step in this action. Could list caches for info if we wanted
+        # main:  julia ${{ github.action_path }}/handle_caches.jl "${{ github.repository }}" "list"
+        main: echo ""
+        post: julia $GITHUB_ACTION_PATH/handle_caches.jl "${{ github.repository }}" "rm" "${{ steps.keys.outputs.restore-key }}"
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+
+    # Windows (because this action uses command prompt on windows)
+    - uses: pyTooling/Actions/with-post-step@adef08d3bdef092282614f3b683897cefae82ee3
+      if: ${{ inputs.delete-old-caches == 'true' && runner.OS == 'Windows' }}
+      with:
+        main: echo ""
+        post: cd %GITHUB_ACTION_PATH% && julia handle_caches.jl "${{ github.repository }}" "rm" "${{ steps.keys.outputs.restore-key }}"
+      env:
+        GH_TOKEN: ${{ inputs.token }}
+
+    - id: hit
+      run: echo "cache-hit=$CACHE_HIT" >> $GITHUB_OUTPUT
       env:
         CACHE_HIT: ${{ steps.cache.outputs.cache-hit }}
       shell: bash

handle_caches.jl

@@ -0,0 +1,65 @@
+using Pkg, Dates
+function handle_caches()
+    repo = ARGS[1]
+    func = ARGS[2]
+    restore_key = get(ARGS, 3, "")
+
+    if func == "list"
+        println("Listing existing caches")
+        run(`gh cache list --limit 100 --repo $repo`)
+    elseif func == "rm"
+        caches = String[]
+        failed = String[]
+        for _ in 1:5 # limit to avoid accidental rate limiting
+            hits = split(strip(read(`gh cache list --limit 100 --repo $repo`, String)), keepempty=false)
+            search_again = length(hits) == 100
+            filter!(contains(restore_key), hits)
+            isempty(hits) && break
+            # We can delete everything that matches the restore key because the new cache is saved later.
+            for c in hits
+                try
+                    run(`gh cache delete $(split(c)[1]) --repo $repo`)
+                    push!(caches, c)
+                catch e
+                    @error e
+                    push!(failed, c)
+                end
+            end
+            search_again || break
+        end
+        if isempty(failed) && isempty(caches)
+            println("No existing caches found for restore key `$restore_key`")
+        else
+            if !isempty(failed)
+                println("Failed to delete $(length(failed)) existing caches for restore key `$restore_key`")
+                println.(failed)
+                @info """
+                    To delete caches you need to grant the following to the default `GITHUB_TOKEN` by adding
+                    this to your yml:
+                    ```
+                    permissions:
+                        actions: write
+                        contents: read
+                    ```
+                    (Note this won't work for fork PRs but should once merged)
+                    Or provide a token with `repo` scope via the `token` input option.
+                    See https://cli.github.com/manual/gh_cache_delete
+                    """
+            end
+            if !isempty(caches)
+                println("$(length(caches)) existing caches deleted that match restore key `$restore_key`:")
+                println.(caches)
+            end
+        end
+    else
+        throw(ArgumentError("Unexpected second argument: $func"))
+    end
+end
+
+try
+    # do a gc with the standard 7-day delay
+    Pkg.gc()
+    handle_caches()
+catch e
+    @error "An error occurred while managing existing caches" e
+end