Update test_harness.jl

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 99
+    labels:
+      - dependencies
+      - github-actions
+

.github/workflows/backup.yml

@@ -1,33 +0,0 @@
-name: Backup
-
-on:
-  schedule:
-    - cron: '5 4 * * 0'
-
-  workflow_dispatch:
-
-jobs:
-  backup:
-    runs-on: ubuntu-20.04
-
-    steps:
-      - name: Configure cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ env.GITHUB_WORKSPACE }}
-            ~/.cache/restic
-          key: ${{ runner.os }}
-
-      - name: Install the correct Python version
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.x'
-
-      - name: Run backup action
-        uses: julia-actions/restic-action@main
-        env: # Options: https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables
-          RESTIC_REPOSITORY: b2:${{ secrets.B2_BUCKET }}:${{ github.repository }}
-          RESTIC_PASSWORD: ${{ secrets.RESTIC_PASSWORD }}
-          B2_ACCOUNT_ID: ${{ secrets.B2_ACCOUNT_ID }}
-          B2_ACCOUNT_KEY: ${{ secrets.B2_ACCOUNT_KEY }}

.github/workflows/ci.yml

@@ -7,17 +7,24 @@ on:
       - "master"
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
     strategy:
       fail-fast: false
       matrix:
         version:
           - "1.0"
+          - "lts"
           - "1" # automatically expands to the latest stable 1.x release of Julia
-          - nightly
+          - "pre"
+          - "nightly"
         os:
           - ubuntu-latest
           - macOS-latest
@@ -25,14 +32,26 @@ jobs:
         arch:
           - x64
           - x86
+          - aarch64
         # 32-bit Julia binaries are not available on macOS
+        # macOS runners use Apple Silicon (ARM64)
         exclude:
           - os: macOS-latest
             arch: x86
+          - os: macOS-latest
+            arch: x64
+          - os: ubuntu-latest
+            arch: aarch64
+          - os: windows-latest
+            arch: aarch64
+          # Julia 1.0 didn't support ARM64
+          - os: macOS-latest
+            arch: aarch64
+            version: "1.0"
 
     steps:
       - name: Checkout Example.jl
-        uses: actions/checkout@v2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           repository: julia-actions/Example.jl
 
@@ -41,35 +60,29 @@ jobs:
         shell: bash
 
       - name: Checkout julia-runtest
-        uses: actions/checkout@v2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           path: ./.github/actions/julia-runtest
 
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
 
-      - uses: actions/cache@v1
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
+      - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0
 
-      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-buildpkg@e3eb439fad4f9aba7da2667e7510e4a46ebc46e1 # v1.7.0
 
       - uses: ./.github/actions/julia-runtest
 
-      - uses: julia-actions/julia-processcoverage@v1
+      - uses: julia-actions/julia-processcoverage@03114f09f119417c3242a9fb6e0b722676aedf38
 
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 # v5.5.1
         with:
-          file: lcov.info
+          files: lcov.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail-ci-if-error: true
+
 
       - name: Verify that test output file exists
         run: test -f "$HOME/julia-runtest"

.github/workflows/test_logger_ci.yml

@@ -7,17 +7,24 @@ on:
       - "master"
   pull_request:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
     strategy:
       fail-fast: false
       matrix:
         version:
           - "1.6"
+          - "lts"
           - "1" # automatically expands to the latest stable 1.x release of Julia
-          - nightly
+          - "pre"
+          - "nightly"
         os:
           - ubuntu-latest
           - macOS-latest
@@ -25,29 +32,32 @@ jobs:
         arch:
           - x64
           - x86
+          - aarch64
         # 32-bit Julia binaries are not available on macOS
+        # macOS runners use Apple Silicon (ARM64)
         exclude:
           - os: macOS-latest
             arch: x86
+          - os: macOS-latest
+            arch: x64
+          - os: ubuntu-latest
+            arch: aarch64
+          - os: windows-latest
+            arch: aarch64
+          # Julia 1.6 didn't support ARM64
+          - os: macOS-latest
+            arch: aarch64
+            version: "1.6"
 
     steps:
       - name: Checkout julia-runtest
-        uses: actions/checkout@v2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2.6.1
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
 
-      - uses: actions/cache@v1
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-
-            ${{ runner.os }}-
+      - uses: julia-actions/cache@d10a6fd8f31b12404a54613ebad242900567f2b9 # v2.1.0
 
       - run: julia --color=yes --check-bounds=yes test_logger_tests.jl

README.md

@@ -18,12 +18,17 @@ on:
       - main
   pull_request:
 
+# needed to allow julia-actions/cache to delete old caches that it has created
+permissions:
+  actions: write
+  contents: read
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        julia-version: ['1.0', '1', 'nightly']
+        julia-version: ['lts', '1', 'pre']
         julia-arch: [x64, x86]
         os: [ubuntu-latest, windows-latest, macOS-latest]
         exclude:
@@ -31,20 +36,23 @@ jobs:
             julia-arch: x86
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: julia-actions/setup-julia@v1
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.julia-version }}
           arch: ${{ matrix.julia-arch }}
+      - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
-        with:
-          annotate: true
+        # with:
+        #   annotate: true
 ```
 
 You can add this workflow to your repository by placing it in a file called `test.yml` in the folder `.github/workflows/`. [More info here](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions).
 
 Here, setting `annotate: true` causes GitHub "annotations" to appear when reviewing the PR, pointing to failing tests, if any.
+This functionality is only enabled on Julia 1.8 (even if `annotate` is set to `true`), since currently it does not work on other Julia versions (see [#76](https://github.com/julia-actions/julia-runtest/issues/76)).
+
 By default, `annotate` is set to false, but that may change in future releases of this action.
 ### Prefixing the Julia command
 
@@ -64,7 +72,7 @@ If you only want to add this prefix on certain builds, you can [include addition
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        version: ['1.0', '1', 'nightly']
+        version: ['lts', '1', 'pre']
         arch: [x64]
         include:
           - os: ubuntu-latest
@@ -78,3 +86,51 @@ If you only want to add this prefix on certain builds, you can [include addition
 ```
 
 This will add the prefix `xvfb-run` to all builds where the `os` is `ubuntu-latest`.
+
+### Pass Arguments to Test Suite
+
+You can pass arguments from the workflow specification to the test script via the `test_args` parameter.
+
+This is useful, for example, to specify separate workflows for fast and slow tests, or conditionally enabling quality assurance tests.
+
+The functionality can be incorporated as follows:
+
+```yaml
+    # ...
+    steps:
+    # ...
+      - uses: julia-actions/julia-runtest@v1
+        with:
+          test_args: 'slow_tests "quality assurance"'
+    # ...
+```
+
+The value of `test_args` can be accessed in `runtest.jl` via the `ARGS` variable. An example for `runtest.jl` is given below.
+
+```julia
+using Test
+# ...
+
+# run fast tests by default
+include("fast_tests.jl")
+
+if "slow_tests" in ARGS
+    # run slow tests
+    include("slow_tests.jl")
+end
+
+if "quality assurance" in ARGS
+    # run quality assurance tests
+    include("qa.jl")
+end
+```
+
+
+### Registry flavor preference
+
+This actions defines (and exports for subsequent steps of the workflow) the
+environmental variable `JULIA_PKG_SERVER_REGISTRY_PREFERENCE=eager` unless it
+is already set. If you want another registry flavor (i.e. `conservative`) this
+should be defined in the `env:` section of the relevant workflow or step. See
+[Registry flavors](https://pkgdocs.julialang.org/dev/registries/#Registry-flavors)
+for more information.

action.yml

@@ -32,29 +32,26 @@ inputs:
   annotate:
     description: 'Whether or not to attempt to create GitHub annotations to show test failures inline. Only effective on Julia 1.8+.'
     default: 'false'
+  compiled_modules:
+    description: 'Whether to run tests with `compiled-modules`. For possible values, refer to https://docs.julialang.org/en/v1/manual/command-line-interface/#command-line-interface'
+    default: 'yes'
+  allow_reresolve:
+    description: 'Whether to allow re-resolving of package versions in the test environment. Only effective on Julia 1.9+. Options: true | false. Default value: true'
+    default: 'true'
+  test_args:
+    description: 'Arguments string that is passed on to test.'
+    default: ''
 
 runs:
   using: 'composite'
   steps:
-    # Occasionally, there are rather large delays (> a few hours)
-    # between the time a package is registered in General and
-    # propagated to pkg.julialang.org.  We can avoid this by manually
-    # cloning ~/.julia/registries/General/ in Julia 1.5 and later.
-    # See:
-    # * https://github.com/JuliaLang/Pkg.jl/issues/2011
-    # * https://github.com/JuliaRegistries/General/issues/16777
-    # * https://github.com/JuliaPackaging/PkgServer.jl/issues/60
-    - run: julia --color=yes "$GITHUB_ACTION_PATH"/add_general_registry.jl
+    - name: Set and export registry flavor preference
+      run: echo "JULIA_PKG_SERVER_REGISTRY_PREFERENCE=${JULIA_PKG_SERVER_REGISTRY_PREFERENCE:-eager}" >> ${GITHUB_ENV}
       shell: bash
-      env:
-        # We set `JULIA_PKG_SERVER` only for this step to enforce
-        # `Pkg.Registry.add` to use Git.  This way, Pkg.jl can send
-        # the request metadata to pkg.julialang.org when installing
-        # packages via `Pkg.test`.
-        JULIA_PKG_SERVER: ""
     - name: Install dependencies in their own (shared) environment
       run: |
-        if VERSION > v"1.8pre"
+        # Functionality only currently works on a narrow range of Julia versions... see #76
+        if v"1.8pre" < VERSION < v"1.9.0-beta3"
           using Pkg
           Pkg.activate("tests-logger-env"; shared=true)
           Pkg.add(Pkg.PackageSpec(name="GitHubActions", version="0.1"))
@@ -63,7 +60,7 @@ runs:
       if: inputs.annotate == 'true'
     - run: |
         # The Julia command that will be executed
-        julia_cmd=( julia --color=yes --depwarn=${{ inputs.depwarn }} --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'include(joinpath(ENV["GITHUB_ACTION_PATH"], "test_harness.jl"))' )
+        julia_cmd=( julia --color=yes --inline=${{ inputs.inline }} --project=${{ inputs.project }} -e 'include(joinpath(ENV["GITHUB_ACTION_PATH"], "test_harness.jl"))' -- ${{inputs.test_args}} )
 
         # Add the prefix in front of the command if there is one
         prefix=( ${{ inputs.prefix }} )
@@ -78,3 +75,6 @@ runs:
         COVERAGE: ${{ inputs.coverage }}
         FORCE_LATEST_COMPATIBLE_VERSION: ${{ inputs.force_latest_compatible_version }}
         CHECK_BOUNDS: ${{ inputs.check_bounds }}
+        COMPILED_MODULES: ${{ inputs.compiled_modules }}
+        ALLOW_RERESOLVE: ${{ inputs.allow_reresolve }}
+        DEPWARN: ${{ inputs.depwarn }}

add_general_registry.jl

@@ -1,49 +0,0 @@
-using Pkg
-
-function general_registry_location()
-    general_registry_dir = joinpath(DEPOT_PATH[1], "registries", "General")
-    registry_toml_file = joinpath(general_registry_dir, "Registry.toml")
-    return general_registry_dir, registry_toml_file
-end
-
-function general_registry_exists()
-    general_registry_dir, registry_toml_file = general_registry_location()
-    if !isdir(general_registry_dir)
-        return false
-    elseif !isfile(registry_toml_file)
-        return false
-    else
-        return true
-    end
-end
-
-function add_general_registry()
-    @info("Attempting to clone the General registry")
-    general_registry_dir, registry_toml_file = general_registry_location()
-    rm(general_registry_dir; force = true, recursive = true)
-    Pkg.Registry.add("General")
-    isfile(registry_toml_file) || throw(ErrorException("the Registry.toml file does not exist"))
-    return nothing
-end
-
-function main(; n = 10, max_delay = 120)
-    VERSION >= v"1.5-" || return
-
-    if general_registry_exists()
-        @info("The General registry already exists locally")
-        return
-    end
-
-    delays = ExponentialBackOff(; n = n, max_delay = max_delay)
-    try
-        retry(add_general_registry; delays = delays)()
-        @info("Successfully added the General registry")
-    catch ex
-        msg = "I was unable to add the General registry. However, the build will continue."
-        @error(msg, exception=(ex,catch_backtrace()))
-    end
-
-    return
-end
-
-main()

kwargs.jl

@@ -6,7 +6,10 @@ include(joinpath(@__DIR__, "autodetect-dependabot.jl"))
 
 function kwargs(; coverage,
                   force_latest_compatible_version,
-                  julia_args::AbstractVector{<:AbstractString}=String[])
+                  allow_reresolve,
+                  julia_args::AbstractVector{<:AbstractString}=String[],
+                  test_args::AbstractVector{<:AbstractString}=String[],
+                  )
     if coverage isa AbstractString
         coverage = parse(Bool, coverage)
     end
@@ -51,6 +54,14 @@ function kwargs(; coverage,
         kwargs_dict[:force_latest_compatible_version] = force_latest_compatible_version::Bool
     end
 
+    if VERSION >= v"1.9"
+        kwargs_dict[:allow_reresolve] = parse(Bool, allow_reresolve)
+    end
+
+    if !isempty(test_args)
+        kwargs_dict[:test_args] = test_args
+    end
+
     return kwargs_dict
 end
 

test_harness.jl

@@ -2,15 +2,90 @@ import Pkg
 include("kwargs.jl")
 kwargs = Kwargs.kwargs(; coverage=ENV["COVERAGE"],
                          force_latest_compatible_version=ENV["FORCE_LATEST_COMPATIBLE_VERSION"],
-                         julia_args=[string("--check-bounds=", ENV["CHECK_BOUNDS"])])
+                         allow_reresolve=ENV["ALLOW_RERESOLVE"],
+                         julia_args=[string("--check-bounds=", ENV["CHECK_BOUNDS"]),
+                                     string("--compiled-modules=", ENV["COMPILED_MODULES"]),
+                                     # Needs to be done via `julia_args` to ensure `depwarn: no` is respected:
+                                     # https://github.com/JuliaLang/Pkg.jl/pull/1763#discussion_r406819660
+                                     string("--depwarn=", ENV["DEPWARN"]),],
+                         test_args=ARGS,
+                         )
 
-if parse(Bool, ENV["ANNOTATE"]) && VERSION > v"1.8pre"
+kwargs_reprs = map(kv -> string(kv[1], "=", repr(kv[2])), collect(kwargs))
+kwargs_repr = join(kwargs_reprs, ", ")
+
+# Warn if running on a merge commit (different from branch HEAD)
+git_note = ""
+if haskey(ENV, "GITHUB_SHA") && get(ENV, "GITHUB_EVENT_NAME", "") == "pull_request" && haskey(ENV, "GITHUB_HEAD_REF")
+    # For pull_request events, GITHUB_SHA is the merge commit, not the PR head commit
+    try
+        merge_commit = ENV["GITHUB_SHA"]
+        pr_branch = ENV["GITHUB_HEAD_REF"]
+        base_branch_name = get(ENV, "GITHUB_BASE_REF", "")
+
+        # Check if there's any difference between the merge commit and the PR head
+        # In GitHub Actions, HEAD^2 is the PR head (second parent of merge commit)
+        # First check if HEAD^2 exists (i.e., this is actually a merge commit)
+        if success(`git rev-parse --verify --quiet HEAD^2`)
+            # Compare tree hashes to check if content actually differs
+            merge_tree = chomp(read(`git rev-parse HEAD^\{tree\}`, String))
+            pr_tree = chomp(read(`git rev-parse HEAD^2^\{tree\}`, String))
+            has_diff = merge_tree != pr_tree
+        else
+            # Not a merge commit, so no difference to report
+            has_diff = false
+        end
+
+        if has_diff
+            base_branch = isempty(base_branch_name) ? "the base branch" : "'$base_branch_name'"
+            global git_note = """
+            │ Note: This is being run on merge commit $merge_commit (merge of PR branch '$pr_branch' into $base_branch).
+            │ The content differs from the actual commit on your PR branch.
+            │ To reproduce locally, update your branch with $base_branch first.
+            │
+            """
+        end
+    catch e
+        @warn "Error while checking git diff" exception=(e, catch_backtrace())
+    end
+end
+
+print("""
+│
+│ To reproduce this CI run locally run the following from the same repository state on julia version $VERSION:
+│
+│ `import Pkg; Pkg.test(;$kwargs_repr)`
+│
+""")
+print(git_note)
+
+if parse(Bool, ENV["ANNOTATE"]) && v"1.8pre" < VERSION < v"1.9.0-beta3"
     push!(LOAD_PATH, "@tests-logger-env") # access dependencies
     using GitHubActions, Logging
     global_logger(GitHubActionsLogger())
     include("test_logger.jl")
     pop!(LOAD_PATH)
+    try
         TestLogger.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
         else
-    Pkg.test(; kwargs...)        
+            rethrow()
+        end
+    end
+else
+    try
+        Pkg.test(; kwargs...)
+    catch e
+        if e isa Pkg.Types.PkgError
+            # don't show the stacktrace of the test harness because it's not useful
+            showerror(stderr, e)
+            exit(1)
+        else
+            rethrow()
+        end
+    end
 end