allow passing multiple arguments in test_args (#118)

Co-authored-by: Markus Hauru <markus@mhauru.org>
Co-authored-by: Ian Butterworth <i.r.butterworth@gmail.com>

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 99
+    labels:
+      - dependencies
+      - github-actions
+

.github/workflows/backup.yml

@@ -1,33 +0,0 @@
-name: Backup
-
-on:
-  schedule:
-    - cron: '5 4 * * 0'
-
-  workflow_dispatch:
-
-jobs:
-  backup:
-    runs-on: ubuntu-20.04
-
-    steps:
-      - name: Configure cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ env.GITHUB_WORKSPACE }}
-            ~/.cache/restic
-          key: ${{ runner.os }}
-
-      - name: Install the correct Python version
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
-
-      - name: Run backup action
-        uses: julia-actions/restic-action@main
-        env: # Options: https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables
-          RESTIC_REPOSITORY: b2:${{ secrets.B2_BUCKET }}:${{ github.repository }}
-          RESTIC_PASSWORD: ${{ secrets.RESTIC_PASSWORD }}
-          B2_ACCOUNT_ID: ${{ secrets.B2_ACCOUNT_ID }}
-          B2_ACCOUNT_KEY: ${{ secrets.B2_ACCOUNT_KEY }}

.github/workflows/ci.yml

@@ -32,7 +32,7 @@ jobs:
 
     steps:
       - name: Checkout Example.jl
-        uses: actions/checkout@v2
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           repository: julia-actions/Example.jl
 
@@ -41,16 +41,16 @@ jobs:
         shell: bash
 
       - name: Checkout julia-runtest
-        uses: actions/checkout@v2
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
         with:
           path: ./.github/actions/julia-runtest
 
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@512e2ab44f5848ea62bddd21d1ee906e989b0360 # v2.4.0
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
 
-      - uses: actions/cache@v1
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
         env:
           cache-name: cache-artifacts
         with:
@@ -61,15 +61,18 @@ jobs:
             ${{ runner.os }}-test-
             ${{ runner.os }}-
 
-      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-buildpkg@90dd6f23eb49626e4e6612cb9d64d456f86e6a1c
 
       - uses: ./.github/actions/julia-runtest
 
-      - uses: julia-actions/julia-processcoverage@v1
+      - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38
 
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673
         with:
           file: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail-ci-if-error: true
+          
 
       - name: Verify that test output file exists
         run: test -f "$HOME/julia-runtest"

.github/workflows/test_logger_ci.yml

@@ -32,14 +32,14 @@ jobs:
 
     steps:
       - name: Checkout julia-runtest
-        uses: actions/checkout@v2
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
 
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@512e2ab44f5848ea62bddd21d1ee906e989b0360 # v2.4.0
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
 
-      - uses: actions/cache@v1
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9
         env:
           cache-name: cache-artifacts
         with:

README.md

@@ -11,19 +11,24 @@ An example workflow that uses this action might look like this:
 ```yaml
 name: Run tests
 
-on: 
+on:
   push:
     branches:
       - master
       - main
   pull_request:
 
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        julia-version: ['1.0', '1', 'nightly']
+        julia-version: ['lts', '1', 'pre']
         julia-arch: [x64, x86]
         os: [ubuntu-latest, windows-latest, macOS-latest]
         exclude:
@@ -31,11 +36,12 @@ jobs:
             julia-arch: x86
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.julia-arch }}
+      - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
         # with:
@@ -45,7 +51,7 @@ jobs:
 You can add this workflow to your repository by placing it in a file called `test.yml` in the folder `.github/workflows/`. [More info here](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions).
 
 Here, setting `annotate: true` causes GitHub "annotations" to appear when reviewing the PR, pointing to failing tests, if any.
-This functionality is only enabled on Julia 1.8 (even if `annotate` is set to `true`), since currently it does not work on other Julia versions (see #76).
+This functionality is only enabled on Julia 1.8 (even if `annotate` is set to `true`), since currently it does not work on other Julia versions (see [#76](https://github.com/julia-actions/julia-runtest/issues/76)).
 
 By default, `annotate` is set to false, but that may change in future releases of this action.
 ### Prefixing the Julia command
@@ -66,7 +72,7 @@ If you only want to add this prefix on certain builds, you can [include addition
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        version: ['1.0', '1', 'nightly']
+        version: ['lts', '1', 'pre']
         arch: [x64]
         include:
           - os: ubuntu-latest
@@ -81,6 +87,44 @@ If you only want to add this prefix on certain builds, you can [include addition
 
 This will add the prefix `xvfb-run` to all builds where the `os` is `ubuntu-latest`.
 
+### Pass Arguments to Test Suite
+
+You can pass arguments from the workflow specification to the test script via the `test_args` parameter.
+
+This is useful, for example, to specify separate workflows for fast and slow tests, or conditionally enabling quality assurance tests.
+
+The functionality can be incorporated as follows:
+
+```yaml
+    # ...
+    steps:
+    # ...
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          test_args: 'slow_tests "quality assurance"'
+    # ...
+```
+
+The value of `test_args` can be accessed in `runtest.jl` via the `ARGS` variable. An example for `runtest.jl` is given below.
+
+```julia
+using Test
+# ...
+
+# run fast tests by default
+include("fast_tests.jl")
+
+if "slow_tests" in ARGS
+    # run slow tests
+    include("slow_tests.jl")
+end
+
+if "quality assurance" in ARGS
+    # run quality assurance tests
+    include("qa.jl")
+end
+```
+
 
 ### Registry flavor preference
 

action.yml

@@ -32,6 +32,15 @@ inputs:
   annotate:
     description: 'Whether or not to attempt to create GitHub annotations to show test failures inline. Only effective on Julia 1.8+.'
     default: 'false'
+  compiled_modules:
+    description: 'Whether to run tests with `compiled-modules`. For possible values, refer to https://docs.julialang.org/en/v1/manual/command-line-interface/#command-line-interface'
+    default: 'yes'
+  allow_reresolve:
+    description: 'Whether to allow re-resolving of package versions in the test environment. Only effective on Julia 1.9+. Options: true | false. Default value: true'
+    default: 'true'
+  test_args:
+    description: 'Arguments string that is passed on to test.'
+    default: ''
 
 runs:
   using: 'composite'
@@ -51,7 +60,7 @@ runs:
       if: inputs.annotate == 'true'
     - run: |
         # The Julia command that will be executed
-        julia_cmd=( julia --color=yes --depwarn=${{ inputs.depwarn }} --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'include(joinpath(ENV["GITHUB_ACTION_PATH"], "test_harness.jl"))' )
+        julia_cmd=( julia --color=yes --depwarn=${{ inputs.depwarn }} --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'include(joinpath(ENV["GITHUB_ACTION_PATH"], "test_harness.jl"))' -- ${{inputs.test_args}} )
 
         # Add the prefix in front of the command if there is one
         prefix=( ${{ inputs.prefix }} )
@@ -66,3 +75,5 @@ runs:
         COVERAGE: ${{ inputs.coverage }}
         FORCE_LATEST_COMPATIBLE_VERSION: ${{ inputs.force_latest_compatible_version }}
         CHECK_BOUNDS: ${{ inputs.check_bounds }}
+        COMPILED_MODULES: ${{ inputs.compiled_modules }}
+        ALLOW_RERESOLVE: ${{ inputs.allow_reresolve }}

kwargs.jl

@@ -6,7 +6,10 @@ include(joinpath(@__DIR__, "autodetect-dependabot.jl"))
 
 function kwargs(; coverage,
                   force_latest_compatible_version,
-                  julia_args::AbstractVector{<:AbstractString}=String[])
+                  allow_reresolve,
+                  julia_args::AbstractVector{<:AbstractString}=String[],
+                  test_args::AbstractVector{<:AbstractString}=String[],
+                  )
     if coverage isa AbstractString
         coverage = parse(Bool, coverage)
     end
@@ -51,6 +54,12 @@ function kwargs(; coverage,
         kwargs_dict[:force_latest_compatible_version] = force_latest_compatible_version::Bool
     end
 
+    if VERSION >= v"1.9"
+        kwargs_dict[:allow_reresolve] = parse(Bool, allow_reresolve)
+    end
+    
+    kwargs_dict[:test_args] = test_args
+
     return kwargs_dict
 end
 

test_harness.jl

@@ -2,15 +2,50 @@ import Pkg
 include("kwargs.jl")
 kwargs = Kwargs.kwargs(; coverage=ENV["COVERAGE"],
                          force_latest_compatible_version=ENV["FORCE_LATEST_COMPATIBLE_VERSION"],
-                         julia_args=[string("--check-bounds=", ENV["CHECK_BOUNDS"])])
+                         allow_reresolve=ENV["ALLOW_RERESOLVE"],
+                         julia_args=[string("--check-bounds=", ENV["CHECK_BOUNDS"]),
+                                     string("--compiled-modules=", ENV["COMPILED_MODULES"])],
+                         test_args=ARGS,
+                         )
 
-if parse(Bool, ENV["ANNOTATE"]) && VERSION > v"1.8pre"
+kwargs_reprs = map(kv -> string(kv[1], "=", repr(kv[2])), collect(kwargs))
+kwargs_repr = join(kwargs_reprs, ", ")
+
+print("""
+│
+│ To reproduce this CI run locally run the following from the same repository state on julia version $VERSION:
+│
+│ `import Pkg; Pkg.test(;$kwargs_repr)`
+│
+""")
+
+if parse(Bool, ENV["ANNOTATE"]) && v"1.8pre" < VERSION < v"1.9.0-beta3"
     push!(LOAD_PATH, "@tests-logger-env") # access dependencies
     using GitHubActions, Logging
     global_logger(GitHubActionsLogger())
     include("test_logger.jl")
     pop!(LOAD_PATH)
-    TestLogger.test(; kwargs...)        
+    try
+        TestLogger.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
 else
-    Pkg.test(; kwargs...)        
+    try
+        Pkg.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
 end